diff options
291 files changed, 11453 insertions, 10385 deletions
diff --git a/AUTHORS.txt b/AUTHORS.txt new file mode 100644 index 0000000..0034abf --- /dev/null +++ b/AUTHORS.txt @@ -0,0 +1,45 @@ +Petteri Aimonen <jpa@npb.mail.kapsi.fi> +Michael Poole <mdpoole@troilus.org> +Daniel Kan <extremeblue99@gmail.com> +Stan Hu <stanhu@aclimalabs.com> +David Hotham <david.hotham@blueyonder.co.uk> +Steffen Siering <steffen siering gmail com> +Jens Steinhauser <jens.steinhauser@gmail.com> +Pavel Ilin <ilin.pa@gmail.com> +Kent Ryhorchuk <kryhorchuk@xeralux.com> +Martin Donath <scifish@gmail.com> +Oliver Lee <oliverzlee@gmail.com> +Michael Haberler <git@mah.priv.at> +Nicolas Colomer <ncolomer@viadeoteam.com> +Ivan Kravets <me@ikravets.com> +Kyle Manna <kyle@kylemanna.com> +Benjamin Kamath <ben.kamath@synapse.com> +Andrew Ruder <andrew.ruder@elecsyscorp.com> +Kenshi Kawaguchi <kenshi@recurse.ca> +isotes <isotes@gmail.com> +Maxim Khitrov <max@mxcrypt.com> +Yaniv Mordekhay <yanivmo@users.noreply.github.com> +Ming Zhao <mzhao@luminatewireless.com> +Google, Inc. +Tom Roeder <tmroeder@google.com> +Piotr Sikora <piotrsikora@google.com> +Bernhard Kraฬmer <bdkrae@gmail.com> +Konstantin Podsvirov <konstantin@podsvirov.pro> +William A. Kennington III <wak@google.com> +Guillaume Lager <g.lager@innoseis.com> +Tobias Haegermarck <tobias.haegermarck@gmail.com> +Justin DeMartino <jdemarti@gmail.com> +Constantine Grantcharov <cgrantcharov@trustpointinnovation.com> +Nick Ewalt <nicholasewalt@google.com> +Harald Fernengel <harryf@gmx.com> +Alice Wang <aw@squareup.com> +Kevin Fitch <kfitch42@gmail.com> +Kamal Marhubi <kamal@marhubi.com> +Elco Jacobs <elco@brewpi.com> +Sรฉbastien Morin <sebastien.morin@primerogames.com> +Dave Flogeras <dflogeras2@gmail.com> +Edward Z. Yang <ezyang@mit.edu> +Robbie Shade <rjshade@google.com> +Andrew Ballinger <andrewballinger@stratisopt.com> +Hamina, Juha-Pekka <Juha-Pekka.Hamina@nordicsemi.no> + @@ -23,6 +23,7 @@ cc_defaults { "-Wno-unused-parameter", ], srcs: [ + "pb_common.c", "pb_decode.c", "pb_encode.c", ], @@ -0,0 +1,21 @@ +licenses(["notice"]) + +exports_files(["LICENSE.txt"]) + +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "nanopb", + visibility = ["//visibility:public"], + hdrs = [ + "pb.h", + "pb_common.h", + "pb_decode.h", + "pb_encode.h", + ], + srcs = [ + "pb_common.c", + "pb_decode.c", + "pb_encode.c", + ], +) diff --git a/CHANGELOG.txt b/CHANGELOG.txt index 6d643ad..f7811a4 100644 --- a/CHANGELOG.txt +++ b/CHANGELOG.txt @@ -1,3 +1,161 @@ +nanopb-0.3.9.1 (2017-04-xx) + Fix handling of special characters in string/bytes default values (issue #322) + Fix encoding of negative numbers with PB_WITHOUT_64BIT (#285) + Fix _zero initializer for enums that don't begin at 0. (#295) + Multiple CMake fixes (#296, #299, #304, #312, #320) + Fix compiler warnings (#305) + Fix scons rules for Python 3 + Add check for large extension field number (issue #306) + Updated included descriptor.proto version (#314) + Resolve oneof sizes symbolically when needed (#311) + Add fixed_count option (#260) + Add some verbose prints in generator (issue #238) + Add test/example of using 'map' type. (#289) + +nanopb-0.3.9 (2017-09-23) + Fix bugs in proto3 encoding of submessages (#256) + Fix message size calculation for arrays of size 1 (#253) + Fix segfault with FT_CALLBACK inside FT_POINTER (#259) + Properly detect truncated tags in corrupted messages (#277) + Make pb_decode_varint32 overflow checks exact (#258) + Add option to build without 64-bit support (#86) + Add options to define source and header file extensions (#264) + Add pb_en/decode_nullterminated() (part of #278) + Add pb_decode_delimited_noinit (#284) + CMake: add dependency for .options file (#265) + CMake: change use of relative paths (#250,#271,#273) + Better error message for missing max_size option (#281) + Travis-CI build fixes (#283) + Add Bazel build system file (#266) + +nanopb-0.3.8 (2017-03-05) + Fix problems with multiple oneofs in same message (#229) + Zero-valued extension fields were mistakenly ignored by encoder (#242) + Multiple fixes related to proto3 mode (#242, #245, #247, #249) + Fix potential unaligned access (#226, #227) + Fix documentation for protoc --plugin argument (#239) + Extend inline / fixed length bytes array support (#244) + Add new option max_length for strings (#107) + Make string substream API more robust (#230) + Make pb_decode_varint32 public API (#231) + Allow overriding proto3 mode (#228) + Add optional enum->string mapping function (#223) + Add transitional options.proto file (#241) + Add better error message on Python library version imcompatibility (#240) + Include version number in PlatformIO library.json (#222) + CMake build script changes (#236, #237) + Change download links to https + Improvements to test cases. + +nanopb-0.3.7 (2016-10-30) + Add support for proto3-style singular fields (#182, #206, #216) + Updated binary package protoc to version 3.1.0 + Add FT_INLINE allocation of bytes fields (#211) + Include package name in include guard (#207) + Fix missing warning with large bytes fields (issue #220) + Added CMake project (#208) + Add bazel BUILD file for nanopb (#209) + Added an AUTHORS file (#211) + Documentation updates + Improvements to test cases. + +nanopb-0.3.6 (2016-06-19) + Protect against corrupted _count fields in pb_release (#205) + Fix error in STATIC_ASSERT with multiple files (#203) + Add -D option to specify output directory (#193) + Generate MIN/MAX/ARRAYSIZE defines for enums (#194) + Generate comments about uncalculable message sizes (#195) + Documentation updates (#196, #201) + Improvements to test cases. + +nanopb-0.3.5 (2016-02-13) + NOTE: If you are using pb_syshdr.h, you will need to add uint_least8_t + definition. See docs/migration.rst for details. + + Fix generator crash with Enum inside Oneof (#188) + Fix some generator regressions related to .options file path (#172) + Add support for platforms without uint8_t (#191) + Allow const parameter to pb_istream_from_buffer (#152) + Ignore null pointers in pb_release() (#183) + Add support for anonymous unions (#184) + Add Python3 support to the generator (#169) + Add code generator insertion points to generated files (#178) + Improvements to CMake script (#181) + Improvements to test cases. + +nanopb-0.3.4 (2015-09-26) + Fix handling of unsigned 8- and 16-bit enums (issue 164) + Fix generator on systems where python = python3. (issue 155) + Fix compiler warning on GCC 5.x (issue 171) + Make the generator better handle imported .protos (issue 165) + Add packed_enum option to generator. + Add syntax= line to .proto files (issue 167) + Add PlatformIO registry manifest file. (pr 156) + +nanopb-0.3.3 (2015-04-10) + Fix missing files in Linux binary package (issue 146) + Fix generator bug when oneof is first field in a message. (issue 142) + Fix generator error when long_names:false is combined with Oneofs. (issue 147) + Fix oneof submessage initialization bug. (issue 149) + Fix problem with plugin options on Python 2.7.2 and older. (issue 153) + Fix crash when callback is inside oneof field. (issue 148) + Switch to .tar.gz format for Mac OS X packages. (issue 154) + Always define enum long names so that cross-file references work. (issue 118) + Add msgid generator option. (issue 151) + Improve comment support in .options files. (issue 145) + Updates for the CMake rule file, add cmake example. + Better error messages for syntax errors in .options file + +nanopb-0.3.2 (2015-01-24) + Fix memory leaks with PB_ENABLE_MALLOC with some submessage hierarchies (issue 138) + Implement support for oneofs (C unions). (issues 131, 141) + Add int_size option for generator (issue 139) + Add compilation option to disable struct packing. (issue 136) + Change PB_RETURN_ERROR() macro to avoid compiler warnings (issue 140) + Fix build problems with protoc 3.0.0 + Add support for POINTER type in extensions + Initialize also extension fields to defaults in pb_decode(). + Detect too large varint values when decoding. + +nanopb-0.3.1 (2014-09-11) + Fix security issue due to size_t overflows. (issue 132) + Fix memory leak with duplicated fields and PB_ENABLE_MALLOC + Fix crash if pb_release() is called twice. + Fix cyclic message support (issue 130) + Fix error in generated initializers for repeated pointer fields. + Improve tests (issues 113, 126) + +nanopb-0.3.0 (2014-08-26) + NOTE: See docs/migration.html or online at + http://koti.kapsi.fi/~jpa/nanopb/docs/migration.html + for changes in this version. Most importantly, you need to add + pb_common.c to the list of files to compile. + + Separated field iterator logic to pb_common.c (issue 128) + Change the _count fields to use pb_size_t datatype (issue 82) + Added PB_ prefix to macro names (issue 106) + Added #if version guard to generated files (issue 129) + Added migration document + +nanopb-0.2.9 (2014-08-09) + NOTE: If you are using the -e option with the generator, you have + to prepend . to the argument to get the same behaviour as before. + + Do not automatically add a dot with generator -e option. (issue 122) + Fix problem with .options file and extension fields. (issue 125) + Don't use SIZE_MAX macro, as it is not in C89. (issue 120) + Generate #defines for initializing message structures. (issue 79) + Add skip_message option to generator. (issue 121) + Add PB_PACKED_STRUCT support for Keil MDK-ARM toolchain (issue 119) + Give better messages about the .options file path. (issue 124) + Improved tests + +nanopb-0.2.8 (2014-05-20) + Fix security issue with PB_ENABLE_MALLOC. (issue 117) + Add option to not add timestamps to .pb.h and .pb.c preambles. (issue 115) + Documentation updates + Improved tests + nanopb-0.2.7 (2014-04-07) Fix bug with default values for extension fields (issue 111) Fix some MISRA-C warnings (issue 91) diff --git a/CMakeLists.txt b/CMakeLists.txt new file mode 100644 index 0000000..2625a66 --- /dev/null +++ b/CMakeLists.txt @@ -0,0 +1,94 @@ +cmake_minimum_required(VERSION 2.8.12) + +project(nanopb C) + +set(nanopb_VERSION_STRING nanopb-0.3.9.1) + +string(REPLACE "nanopb-" "" nanopb_VERSION ${nanopb_VERSION_STRING}) + +option(nanopb_BUILD_RUNTIME "Build the headers and libraries needed at runtime" ON) +option(nanopb_BUILD_GENERATOR "Build the protoc plugin for code generation" ON) +option(nanopb_MSVC_STATIC_RUNTIME "Link static runtime libraries" ON) + +if(NOT DEFINED nanopb_PROTOC_PATH) + set(nanopb_PROTOC_PATH "protoc") +endif() + +if(NOT DEFINED CMAKE_DEBUG_POSTFIX) + set(CMAKE_DEBUG_POSTFIX "d") +endif() + +include(GNUInstallDirs) + +if(MSVC AND nanopb_MSVC_STATIC_RUNTIME) + foreach(flag_var + CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELEASE + CMAKE_C_FLAGS_MINSIZEREL CMAKE_C_FLAGS_RELWITHDEBINFO) + if(${flag_var} MATCHES "/MD") + string(REGEX REPLACE "/MD" "/MT" ${flag_var} "${${flag_var}}") + endif(${flag_var} MATCHES "/MD") + endforeach(flag_var) +endif() + +if(NOT DEFINED CMAKE_INSTALL_CMAKEDIR) + set(CMAKE_INSTALL_CMAKEDIR "lib/cmake/nanopb") +endif() + +if(nanopb_BUILD_GENERATOR) + set(generator_protos nanopb plugin) + + find_package(PythonInterp 2.7 REQUIRED) + execute_process( + COMMAND ${PYTHON_EXECUTABLE} -c + "from distutils import sysconfig; print(sysconfig.get_python_lib(prefix=''))" + OUTPUT_VARIABLE PYTHON_INSTDIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + foreach(generator_proto IN LISTS generator_protos) + string(REGEX REPLACE "([^;]+)" "${PROJECT_SOURCE_DIR}/generator/proto/\\1.proto" generator_proto_file "${generator_proto}") + string(REGEX REPLACE "([^;]+)" "\\1_pb2.py" generator_proto_py_file "${generator_proto}") + add_custom_command( + OUTPUT ${generator_proto_py_file} + COMMAND ${nanopb_PROTOC_PATH} --python_out=${PROJECT_BINARY_DIR} -I${PROJECT_SOURCE_DIR}/generator/proto ${generator_proto_file} + DEPENDS ${generator_proto_file} + ) + add_custom_target("generate_${generator_proto_py_file}" ALL DEPENDS ${generator_proto_py_file}) + install( + FILES ${PROJECT_BINARY_DIR}/${generator_proto_py_file} + DESTINATION ${PYTHON_INSTDIR} + ) + endforeach() +endif() + +if(nanopb_BUILD_RUNTIME) + add_library(protobuf-nanopb STATIC + pb.h + pb_common.h + pb_common.c + pb_encode.h + pb_encode.c + pb_decode.h + pb_decode.c) + + target_include_directories(protobuf-nanopb INTERFACE + $<INSTALL_INTERFACE:${CMAKE_INSTALL_INCLUDEDIR}> + ) + + configure_file(extra/nanopb-config-version.cmake.in + nanopb-config-version.cmake @ONLY) + + install(TARGETS protobuf-nanopb EXPORT nanopb-targets + ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR}) + + install(EXPORT nanopb-targets + DESTINATION ${CMAKE_INSTALL_CMAKEDIR} + NAMESPACE nanopb::) + + install(FILES extra/nanopb-config.cmake + ${CMAKE_CURRENT_BINARY_DIR}/nanopb-config-version.cmake + DESTINATION ${CMAKE_INSTALL_CMAKEDIR}) + + install(FILES pb.h pb_common.h pb_encode.h pb_decode.h + DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}) +endif() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..4041bc3 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,32 @@ +Contributing to Nanopb development +================================== + +Reporting issues and requesting features +---------------------------------------- + +Feel free to report any issues you see or features you would like +to see in the future to the Github issue tracker. Using the templates +below is preferred: + +* [Report a bug](https://github.com/nanopb/nanopb/issues/new?body=**Steps%20to%20reproduce%20the%20issue**%0a%0a1.%0a2.%0a3.%0a%0a**What%20happens?**%0A%0A**What%20should%20happen?**&labels=Type-Defect) +* [Request a feature](https://github.com/nanopb/nanopb/issues/new?body=**What%20should%20the%20feature%20do?**%0A%0A**In%20what%20situation%20would%20the%20feature%20be%20useful?**&labels=Type-Enhancement) + +Requesting help +--------------- + +If there is something strange going on, but you do not know if +it is actually a bug in nanopb, try asking first on the +[discussion forum](https://groups.google.com/forum/#!forum/nanopb). + +Pull requests +------------- + +Pull requests are welcome! + +If it is not obvious from the commit message, please indicate the +same information as you would for an issue report: + +* What functionality it fixes/adds. +* How can the problem be reproduced / when would the feature be useful. + + diff --git a/README.android b/README.android index 2a2d518..5ce0117 100644 --- a/README.android +++ b/README.android @@ -1,5 +1,5 @@ -URL: http://koti.kapsi.fi/jpa/nanopb/ -Version: 2.8.0 +URL: https://jpa.kapsi.fi/nanopb/ +Version: 0.3.9.1 License: zlib Description: "nanopb-c: A plugin for Google Protobuf compiler that generates C code" @@ -9,4 +9,4 @@ LOCAL_SRC_FILES and set LOCAL_PROTOC_OPTIMIZE_TYPE: LOCAL_SRC_FILES += simple.proto LOCAL_PROTOC_OPTIMIZE_TYPE := nanopb-c -Then look at http://koti.kapsi.fi/jpa/nanopb/ Documentation for how to use nanopb-c. +Then look at https://jpa.kapsi.fi/nanopb/ Documentation for how to use nanopb-c. @@ -1,18 +1,25 @@ +Nanopb - Protocol Buffers for Embedded Systems +============================================== + +[](https://travis-ci.org/nanopb/nanopb) + Nanopb is a small code-size Protocol Buffers implementation in ansi C. It is especially suitable for use in microcontrollers, but fits any memory restricted system. -Homepage: http://kapsi.fi/~jpa/nanopb/ - +* **Homepage:** https://jpa.kapsi.fi/nanopb/ +* **Documentation:** https://jpa.kapsi.fi/nanopb/docs/ +* **Downloads:** https://jpa.kapsi.fi/nanopb/download/ +* **Forum:** https://groups.google.com/forum/#!forum/nanopb Using the nanopb library -======================== +------------------------ To use the nanopb library, you need to do two things: -1) Compile your .proto files for nanopb, using protoc. -2) Include pb_encode.c and pb_decode.c in your project. +1. Compile your .proto files for nanopb, using protoc. +2. Include pb_encode.c, pb_decode.c and pb_common.c in your project. The easiest way to get started is to study the project in "examples/simple". It contains a Makefile, which should work directly under most Linux systems. @@ -22,7 +29,7 @@ README.txt in that folder. Using the Protocol Buffers compiler (protoc) -============================================ +-------------------------------------------- The nanopb generator is implemented as a plugin for the Google's own protoc compiler. This has the advantage that there is no need to reimplement the basic parsing of .proto files. However, it does mean that you need the @@ -46,7 +53,7 @@ protoc, you need to manually give the path to nanopb generator: Running the tests -================= +----------------- If you want to perform further development of the nanopb core, or to verify its functionality using your compiler and platform, you'll want to run the test suite. The build rules for the test suite are implemented using Scons, @@ -58,4 +65,7 @@ so you need to have that installed. To run the tests: This will show the progress of various test cases. If the output does not end in an error, the test cases were successful. - +Note: Mac OS X by default aliases 'clang' as 'gcc', while not actually +supporting the same command line options as gcc does. To run tests on +Mac OS X, use: "scons CC=clang CXX=clang". Same way can be used to run +tests with different compilers on any platform. diff --git a/README.version b/README.version index 837b3ef..b0e5805 100644 --- a/README.version +++ b/README.version @@ -1,3 +1,3 @@ -URL: http://koti.kapsi.fi/~jpa/nanopb/download/nanopb-0.2.8.tar.gz -Version: 0.2.8 +URL: https://github.com/nanopb/nanopb/archive/0.3.9.1.tar.gz +Version: 0.3.9.1 BugComponent: 31808 diff --git a/WORKSPACE b/WORKSPACE new file mode 100644 index 0000000..2837cb3 --- /dev/null +++ b/WORKSPACE @@ -0,0 +1 @@ +workspace(name="com_github_nanopb_nanopb") diff --git a/docs/Makefile b/docs/Makefile index e7143b2..0dbd97c 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -1,4 +1,4 @@ -all: index.html concepts.html reference.html security.html \ +all: index.html concepts.html reference.html security.html migration.html \ generator_flow.png %.png: %.svg diff --git a/docs/concepts.rst b/docs/concepts.rst index 0df5ad6..1f9aec1 100644 --- a/docs/concepts.rst +++ b/docs/concepts.rst @@ -148,7 +148,9 @@ Most Protocol Buffers datatypes have directly corresponding C datatypes, such as 1) Strings, bytes and repeated fields of any type map to callback functions by default. 2) If there is a special option *(nanopb).max_size* specified in the .proto file, string maps to null-terminated char array and bytes map to a structure containing a char array and a size field. -3) If there is a special option *(nanopb).max_count* specified on a repeated field, it maps to an array of whatever type is being repeated. Another field will be created for the actual number of entries stored. +3) If *(nanopb).fixed_length* is set to *true* and *(nanopb).max_size* is also set, then bytes map to an inline byte array of fixed size. +4) If there is a special option *(nanopb).max_count* specified on a repeated field, it maps to an array of whatever type is being repeated. Another field will be created for the actual number of entries stored. +5) If *(nanopb).fixed_count* is set to *true* and *(nanopb).max_count* is also set, the field for the actual number of entries will not by created as the count is always assumed to be max count. =============================================================================== ======================= field in .proto autogenerated in .h @@ -160,16 +162,22 @@ repeated string name = 1 [(nanopb).max_size = 40, (nanopb).max_count = 5]; | char name[5][40]; required bytes data = 1 [(nanopb).max_size = 40]; | typedef struct { | size_t size; - | uint8_t bytes[40]; + | pb_byte_t bytes[40]; | } Person_data_t; | Person_data_t data; +required bytes data = 1 [(nanopb).max_size = 40, (nanopb).fixed_length = true]; | pb_byte_t data[40]; +repeated int32 data = 1 [(nanopb).max_count = 5, (nanopb).fixed_count true]; | int32_t data[5]; =============================================================================== ======================= The maximum lengths are checked in runtime. If string/bytes/array exceeds the allocated length, *pb_decode* will return false. -Note: for the *bytes* datatype, the field length checking may not be exact. +Note: For the *bytes* datatype, the field length checking may not be exact. The compiler may add some padding to the *pb_bytes_t* structure, and the nanopb runtime doesn't know how much of the structure size is padding. Therefore it uses the whole length of the structure for storing data, which is not very smart but shouldn't cause problems. In practise, this means that if you specify *(nanopb).max_size=5* on a *bytes* field, you may be able to store 6 bytes there. For the *string* field type, the length limit is exact. +Note: When using the *fixed_count* option, the decoder assumes the repeated elements are +received sequentially or that repeated elements for a non-packed field will not be interleaved with +another *fixed_count* non-packed field. + Field callbacks =============== When a field has dynamic length, nanopb cannot statically allocate storage for it. Instead, it allows you to handle the field in whatever way you want, using a callback function. @@ -255,6 +263,61 @@ generates this field description array for the structure *Person_PhoneNumber*:: PB_LAST_FIELD }; +Oneof +===== +Protocol Buffers supports `oneof`_ sections. Here is an example of ``oneof`` usage:: + + message MsgType1 { + required int32 value = 1; + } + + message MsgType2 { + required bool value = 1; + } + + message MsgType3 { + required int32 value1 = 1; + required int32 value2 = 2; + } + + message MyMessage { + required uint32 uid = 1; + required uint32 pid = 2; + required uint32 utime = 3; + + oneof payload { + MsgType1 msg1 = 4; + MsgType2 msg2 = 5; + MsgType3 msg3 = 6; + } + } + +Nanopb will generate ``payload`` as a C union and add an additional field ``which_payload``:: + + typedef struct _MyMessage { + uint32_t uid; + uint32_t pid; + uint32_t utime; + pb_size_t which_payload; + union { + MsgType1 msg1; + MsgType2 msg2; + MsgType3 msg3; + } payload; + /* @@protoc_insertion_point(struct:MyMessage) */ + } MyMessage; + +``which_payload`` indicates which of the ``oneof`` fields is actually set. +The user is expected to set the filed manually using the correct field tag:: + + MyMessage msg = MyMessage_init_zero; + msg.payload.msg2.value = true; + msg.which_payload = MyMessage_msg2_tag; + +Notice that neither ``which_payload`` field nor the unused fileds in ``payload`` +will consume any space in the resulting encoded message. + +.. _`oneof`: https://developers.google.com/protocol-buffers/docs/reference/proto2-spec#oneof_and_oneof_field Extension fields ================ @@ -299,6 +362,44 @@ An example of this is available in *tests/test_encode_extensions.c* and .. _`extension fields`: https://developers.google.com/protocol-buffers/docs/proto#extensions +Default values +============== +Protobuf has two syntax variants, proto2 and proto3. Of these proto2 has user +definable default values that can be given in .proto file:: + + message MyMessage { + optional bytes foo = 1 [default = "ABC\x01\x02\x03"]; + optional string bar = 2 [default = "รฅรครถ"]; + } + +Nanopb will generate both static and runtime initialization for the default +values. In `myproto.pb.h` there will be a `#define MyMessage_init_default` that +can be used to initialize whole message into default values:: + + MyMessage msg = MyMessage_init_default; + +In addition to this, `pb_decode()` will initialize message fields to defaults +at runtime. If this is not desired, `pb_decode_noinit()` can be used instead. + +Message framing +=============== +Protocol Buffers does not specify a method of framing the messages for transmission. +This is something that must be provided by the library user, as there is no one-size-fits-all +solution. Typical needs for a framing format are to: + +1. Encode the message length. +2. Encode the message type. +3. Perform any synchronization and error checking that may be needed depending on application. + +For example UDP packets already fullfill all the requirements, and TCP streams typically only +need a way to identify the message length and type. Lower level interfaces such as serial ports +may need a more robust frame format, such as HDLC (high-level data link control). + +Nanopb provides a few helpers to facilitate implementing framing formats: + +1. Functions *pb_encode_delimited* and *pb_decode_delimited* prefix the message data with a varint-encoded length. +2. Union messages and oneofs are supported in order to implement top-level container messages. +3. Message IDs can be specified using the *(nanopb_msgopt).msgid* option and can then be accessed from the header. Return values and error handling ================================ @@ -312,5 +413,5 @@ The error messages help in guessing what is the underlying cause of the error. T 3) IO errors in your own stream callbacks. 4) Errors that happen in your callback functions. 5) Exceeding the max_size or bytes_left of a stream. -6) Exceeding the max_size of a string or array field +6) Exceeding the max_size/max_count of a string or array field 7) Invalid protocol buffers binary message. diff --git a/docs/index.rst b/docs/index.rst index d49abc0..afc7ee4 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -7,7 +7,7 @@ Nanopb: Protocol Buffers with small code size Nanopb is an ANSI-C library for encoding and decoding messages in Google's `Protocol Buffers`__ format with minimal requirements for RAM and code space. It is primarily suitable for 32-bit microcontrollers. -__ http://code.google.com/apis/protocolbuffers/ +__ https://developers.google.com/protocol-buffers/docs/reference/overview Overall structure ================= @@ -23,6 +23,7 @@ So a typical project might include these files: 1) Nanopb runtime library: - pb.h + - pb_common.h and pb_common.c (always needed) - pb_decode.h and pb_decode.c (needed for decoding messages) - pb_encode.h and pb_encode.c (needed for encoding messages) 2) Protocol description (you can have many): @@ -39,9 +40,9 @@ Features and limitations #) Small code size (2โ10 kB depending on processor, plus any message definitions) #) Small ram usage (typically ~300 bytes, plus any message structs) #) Allows specifying maximum size for strings and arrays, so that they can be allocated statically. -#) No malloc needed: everything can be allocated statically or on the stack. +#) No malloc needed: everything can be allocated statically or on the stack. Optional malloc support available. #) You can use either encoder or decoder alone to cut the code size in half. -#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, packed arrays, extension fields. +#) Support for most protobuf features, including: all data types, nested submessages, default values, repeated and optional fields, oneofs, packed arrays, extension fields. #) Callback mechanism for handling messages larger than can fit in available RAM. #) Extensive set of tests. @@ -53,8 +54,8 @@ Features and limitations #) Fields in the generated structs are ordered by the tag number, instead of the natural ordering in .proto file. #) Unknown fields are not preserved when decoding and re-encoding a message. #) Reflection (runtime introspection) is not supported. E.g. you can't request a field by giving its name in a string. -#) Numeric arrays are always encoded as packed, even if not marked as packed in .proto. This causes incompatibility with decoders that do not support packed format. -#) Cyclic references between messages are supported only in callback mode. +#) Numeric arrays are always encoded as packed, even if not marked as packed in .proto. +#) Cyclic references between messages are supported only in callback and malloc mode. Getting started =============== diff --git a/docs/menu.rst b/docs/menu.rst index a49b22c..2c110de 100644 --- a/docs/menu.rst +++ b/docs/menu.rst @@ -4,8 +4,10 @@ 2) `Concepts`_ 3) `API reference`_ 4) `Security model`_ + 5) `Migration from older versions`_ .. _`Overview`: index.html .. _`Concepts`: concepts.html .. _`API reference`: reference.html .. _`Security model`: security.html +.. _`Migration from older versions`: migration.html diff --git a/docs/migration.rst b/docs/migration.rst new file mode 100644 index 0000000..fdc1d8c --- /dev/null +++ b/docs/migration.rst @@ -0,0 +1,320 @@ +===================================== +Nanopb: Migration from older versions +===================================== + +.. include :: menu.rst + +This document details all the breaking changes that have been made to nanopb +since its initial release. For each change, the rationale and required +modifications of user applications are explained. Also any error indications +are included, in order to make it easier to find this document. + +.. contents :: + +Nanopb-0.3.9.1, 0.4.0 (2018-xx-xx) +================================== + +Fix handling of string and bytes default values +----------------------------------------------- + +**Rationale:** Previously nanopb didn't properly decode special character +escapes like \\200 emitted by protoc. This caused these escapes to end up +verbatim in the default values in .pb.c file. + +**Changes:** Escapes are now decoded, and e.g. "\\200" or "\\x80" results in +{0x80} for bytes field and "\\x80" for string field. + +**Required actions:** If code has previously relied on '\\' in default value +being passed through verbatim, it must now be changed to '\\\\'. + +Nanopb-0.3.8 (2017-03-05) +========================= + +Fully drain substreams before closing +------------------------------------- + +**Rationale:** If the substream functions were called directly and the caller +did not completely empty the substring before closing it, the parent stream +would be put into an incorrect state. + +**Changes:** *pb_close_string_substream* can now error and returns a boolean. + +**Required actions:** Add error checking onto any call to +*pb_close_string_substream*. + +Change oneof format in .pb.c files +---------------------------------- + +**Rationale:** Previously two oneofs in a single message would be erroneously +handled as part of the same union. + +**Changes:** Oneofs fields now use special *PB_DATAOFFSET_UNION* offset type +in generated .pb.c files to distinguish whether they are the first or following +field inside an union. + +**Required actions:** Regenerate *.pb.c/.pb.h* files with new nanopb version if +oneofs are used. + +Nanopb-0.3.5 (2016-02-13) +========================= + +Add support for platforms without uint8_t +----------------------------------------- +**Rationale:** Some platforms cannot access 8-bit sized values directly, and +do not define *uint8_t*. Nanopb previously didn't support these platforms. + +**Changes:** References to *uint8_t* were replaced with several alternatives, +one of them being a new *pb_byte_t* typedef. This in turn uses *uint_least8_t* +which means the smallest available type. + +**Required actions:** If your platform does not have a standards-compliant +*stdint.h*, it may lack the definition for *[u]int_least8_t*. This must be +added manually, example can be found in *extra/pb_syshdr.h*. + +**Error indications:** Compiler error: "unknown type name 'uint_least8_t'". + +Nanopb-0.3.2 (2015-01-24) +========================= + +Add support for OneOfs +---------------------- +**Rationale:** Previously nanopb did not support the *oneof* construct in +*.proto* files. Those fields were generated as regular *optional* fields. + +**Changes:** OneOfs are now generated as C unions. Callback fields are not +supported inside oneof and generator gives an error. + +**Required actions:** The generator option *no_unions* can be used to restore old +behaviour and to allow callbacks to be used. To use unions, one change is +needed: use *which_xxxx* field to detect which field is present, instead +of *has_xxxx*. Compare the value against *MyStruct_myfield_tag*. + +**Error indications:** Generator error: "Callback fields inside of oneof are +not supported". Compiler error: "Message" has no member named "has_xxxx". + +Nanopb-0.3.0 (2014-08-26) +========================= + +Separate field iterator logic to pb_common.c +-------------------------------------------- +**Rationale:** Originally, the field iteration logic was simple enough to be +duplicated in *pb_decode.c* and *pb_encode.c*. New field types have made the +logic more complex, which required the creation of a new file to contain the +common functionality. + +**Changes:** There is a new file, *pb_common.c*, which must be included in +builds. + +**Required actions:** Add *pb_common.c* to build rules. This file is always +required. Either *pb_decode.c* or *pb_encode.c* can still be left out if some +functionality is not needed. + +**Error indications:** Linker error: undefined reference to +*pb_field_iter_begin*, *pb_field_iter_next* or similar. + +Change data type of field counts to pb_size_t +--------------------------------------------- +**Rationale:** Often nanopb is used with small arrays, such as 255 items or +less. Using a full *size_t* field to store the array count wastes memory if +there are many arrays. There already exists parameters *PB_FIELD_16BIT* and +*PB_FIELD_32BIT* which tell nanopb what is the maximum size of arrays in use. + +**Changes:** Generator will now use *pb_size_t* for the array *_count* fields. +The size of the type will be controlled by the *PB_FIELD_16BIT* and +*PB_FIELD_32BIT* compilation time options. + +**Required actions:** Regenerate all *.pb.h* files. In some cases casts to the +*pb_size_t* type may need to be added in the user code when accessing the +*_count* fields. + +**Error indications:** Incorrect data at runtime, crashes. But note that other +changes in the same version already require regenerating the files and have +better indications of errors, so this is only an issue for development +versions. + +Renamed some macros and identifiers +----------------------------------- +**Rationale:** Some names in nanopb core were badly chosen and conflicted with +ISO C99 reserved names or lacked a prefix. While they haven't caused trouble +so far, it is reasonable to switch to non-conflicting names as these are rarely +used from user code. + +**Changes:** The following identifier names have changed: + + * Macros: + + * STATIC_ASSERT(x) -> PB_STATIC_ASSERT(x) + * UNUSED(x) -> PB_UNUSED(x) + + * Include guards: + + * _PB_filename_ -> PB_filename_INCLUDED + + * Structure forward declaration tags: + + * _pb_field_t -> pb_field_s + * _pb_bytes_array_t -> pb_bytes_array_s + * _pb_callback_t -> pb_callback_s + * _pb_extension_type_t -> pb_extension_type_s + * _pb_extension_t -> pb_extension_s + * _pb_istream_t -> pb_istream_s + * _pb_ostream_t -> pb_ostream_s + +**Required actions:** Regenerate all *.pb.c* files. If you use any of the above +identifiers in your application code, perform search-replace to the new name. + +**Error indications:** Compiler errors on lines with the macro/type names. + +Nanopb-0.2.9 (2014-08-09) +========================= + +Change semantics of generator -e option +--------------------------------------- +**Rationale:** Some compilers do not accept filenames with two dots (like +in default extension .pb.c). The *-e* option to the generator allowed changing +the extension, but not skipping the extra dot. + +**Changes:** The *-e* option in generator will no longer add the prepending +dot. The default value has been adjusted accordingly to *.pb.c* to keep the +default behaviour the same as before. + +**Required actions:** Only if using the generator -e option. Add dot before +the parameter value on the command line. + +**Error indications:** File not found when trying to compile generated files. + +Nanopb-0.2.7 (2014-04-07) +========================= + +Changed pointer-type bytes field datatype +----------------------------------------- +**Rationale:** In the initial pointer encoding support since nanopb-0.2.5, +the bytes type used a separate *pb_bytes_ptr_t* type to represent *bytes* +fields. This made it easy to encode data from a separate, user-allocated +buffer. However, it made the internal logic more complex and was inconsistent +with the other types. + +**Changes:** Dynamically allocated bytes fields now have the *pb_bytes_array_t* +type, just like statically allocated ones. + +**Required actions:** Only if using pointer-type fields with the bytes datatype. +Change any access to *msg->field.size* to *msg->field->size*. Change any +allocation to reserve space of amount *PB_BYTES_ARRAY_T_ALLOCSIZE(n)*. If the +data pointer was begin assigned from external source, implement the field using +a callback function instead. + +**Error indications:** Compiler error: unknown type name *pb_bytes_ptr_t*. + +Nanopb-0.2.4 (2013-11-07) +========================= + +Remove the NANOPB_INTERNALS compilation option +---------------------------------------------- +**Rationale:** Having the option in the headers required the functions to +be non-static, even if the option is not used. This caused errors on some +static analysis tools. + +**Changes:** The *#ifdef* and associated functions were removed from the +header. + +**Required actions:** Only if the *NANOPB_INTERNALS* option was previously +used. Actions are as listed under nanopb-0.1.3 and nanopb-0.1.6. + +**Error indications:** Compiler warning: implicit declaration of function +*pb_dec_string*, *pb_enc_string*, or similar. + +Nanopb-0.2.1 (2013-04-14) +========================= + +Callback function signature +--------------------------- +**Rationale:** Previously the auxilary data to field callbacks was passed +as *void\**. This allowed passing of any data, but made it unnecessarily +complex to return a pointer from callback. + +**Changes:** The callback function parameter was changed to *void\*\**. + +**Required actions:** You can continue using the old callback style by +defining *PB_OLD_CALLBACK_STYLE*. Recommended action is to: + + * Change the callback signatures to contain *void\*\** for decoders and + *void \* const \** for encoders. + * Change the callback function body to use *\*arg* instead of *arg*. + +**Error indications:** Compiler warning: assignment from incompatible +pointer type, when initializing *funcs.encode* or *funcs.decode*. + +Nanopb-0.2.0 (2013-03-02) +========================= + +Reformatted generated .pb.c file using macros +--------------------------------------------- +**Rationale:** Previously the generator made a list of C *pb_field_t* +initializers in the .pb.c file. This led to a need to regenerate all .pb.c +files after even small changes to the *pb_field_t* definition. + +**Changes:** Macros were added to pb.h which allow for cleaner definition +of the .pb.c contents. By changing the macro definitions, changes to the +field structure are possible without breaking compatibility with old .pb.c +files. + +**Required actions:** Regenerate all .pb.c files from the .proto sources. + +**Error indications:** Compiler warning: implicit declaration of function +*pb_delta_end*. + +Changed pb_type_t definitions +----------------------------- +**Rationale:** The *pb_type_t* was previously an enumeration type. This +caused warnings on some compilers when using bitwise operations to set flags +inside the values. + +**Changes:** The *pb_type_t* was changed to *typedef uint8_t*. The values +were changed to *#define*. Some value names were changed for consistency. + +**Required actions:** Only if you directly access the `pb_field_t` contents +in your own code, something which is not usually done. Needed changes: + + * Change *PB_HTYPE_ARRAY* to *PB_HTYPE_REPEATED*. + * Change *PB_HTYPE_CALLBACK* to *PB_ATYPE()* and *PB_ATYPE_CALLBACK*. + +**Error indications:** Compiler error: *PB_HTYPE_ARRAY* or *PB_HTYPE_CALLBACK* +undeclared. + +Nanopb-0.1.6 (2012-09-02) +========================= + +Refactored field decoder interface +---------------------------------- +**Rationale:** Similarly to field encoders in nanopb-0.1.3. + +**Changes:** New functions with names *pb_decode_\** were added. + +**Required actions:** By defining NANOPB_INTERNALS, you can still keep using +the old functions. Recommended action is to replace any calls with the newer +*pb_decode_\** equivalents. + +**Error indications:** Compiler warning: implicit declaration of function +*pb_dec_string*, *pb_dec_varint*, *pb_dec_submessage* or similar. + +Nanopb-0.1.3 (2012-06-12) +========================= + +Refactored field encoder interface +---------------------------------- +**Rationale:** The old *pb_enc_\** functions were designed mostly for the +internal use by the core. Because they are internally accessed through +function pointers, their signatures had to be common. This led to a confusing +interface for external users. + +**Changes:** New functions with names *pb_encode_\** were added. These have +easier to use interfaces. The old functions are now only thin wrappers for +the new interface. + +**Required actions:** By defining NANOPB_INTERNALS, you can still keep using +the old functions. Recommended action is to replace any calls with the newer +*pb_encode_\** equivalents. + +**Error indications:** Compiler warning: implicit declaration of function +*pb_enc_string*, *pb_enc_varint, *pb_enc_submessage* or similar. + diff --git a/docs/reference.rst b/docs/reference.rst index ccbf0a4..40a69ad 100644 --- a/docs/reference.rst +++ b/docs/reference.rst @@ -20,10 +20,9 @@ You must have the same settings for the nanopb library and all code that includes pb.h. ============================ ================================================ -__BIG_ENDIAN__ Set this if your platform stores integers and - floats in big-endian format. Mixed-endian - systems (different layout for ints and floats) - are currently not supported. +PB_NO_PACKED_STRUCTS Disable packed structs. Increases RAM usage but + is necessary on some platforms that do not + support unaligned memory access. PB_ENABLE_MALLOC Set this to enable dynamic allocation support in the decoder. PB_MAX_REQUIRED_FIELDS Maximum number of required fields to check for @@ -54,6 +53,8 @@ PB_SYSTEM_HEADER Replace the standard header files with a single functions and typedefs listed on the `overview page`_. Value must include quotes, for example *#define PB_SYSTEM_HEADER "foo.h"*. +PB_WITHOUT_64BIT Disable 64-bit support, for old compilers or + for a slight speedup on 8-bit platforms. ============================ ================================================ The PB_MAX_REQUIRED_FIELDS, PB_FIELD_16BIT and PB_FIELD_32BIT settings allow @@ -73,18 +74,31 @@ The generator behaviour can be adjusted using these options, defined in the max_size Allocated size for *bytes* and *string* fields. max_count Allocated number of entries in arrays (*repeated* fields). +int_size Override the integer type of a field. + (To use e.g. uint8_t to save RAM.) type Type of the generated field. Default value is *FT_DEFAULT*, which selects automatically. You can use *FT_CALLBACK*, *FT_POINTER*, - *FT_STATIC* or *FT_IGNORE* to force a callback - field, a dynamically allocated field, a static - field or to completely ignore the field. + *FT_STATIC* or *FT_IGNORE* to + force a callback field, a dynamically + allocated field, a static field or to + completely ignore the field. long_names Prefix the enum name to the enum value in definitions, i.e. *EnumName_EnumValue*. Enabled by default. packed_struct Make the generated structures packed. NOTE: This cannot be used on CPUs that break on unaligned accesses to variables. +skip_message Skip the whole message from generation. +no_unions Generate 'oneof' fields as optional fields + instead of C unions. +msgid Specifies a unique id for this message type. + Can be used by user code as an identifier. +anonymous_oneof Generate 'oneof' fields as anonymous unions. +fixed_length Generate 'bytes' fields with constant length + (max_size must also be defined). +fixed_count Generate arrays with constant length + (max_count must also be defined). ============================ ================================================ These options can be defined for the .proto files before they are converted @@ -143,8 +157,20 @@ options from it. The file format is as follows: it makes sense to define wildcard options first in the file and more specific ones later. -If preferred, the name of the options file can be set using the command line -switch *-f* to nanopb_generator.py. +To debug problems in applying the options, you can use the *-v* option for the +plugin. Plugin options are specified in front of the output path: + + protoc ... --nanopb_out=-v:. message.proto + +Protoc doesn't currently pass include path into plugins. Therefore if your +*.proto* is in a subdirectory, nanopb may have trouble finding the associated +*.options* file. A workaround is to specify include path separately to the +nanopb plugin, like: + + protoc -Isubdir --nanopb_out=-Isubdir:. message.proto + +If preferred, the name of the options file can be set using plugin argument +*-f*. Defining the options on command line ------------------------------------ @@ -170,7 +196,7 @@ nanopb.proto can be found. This file, in turn, requires the file */usr/include*. Therefore, to compile a .proto file which uses options, use a protoc command similar to:: - protoc -I/usr/include -Inanopb/generator -I. -omessage.pb message.proto + protoc -I/usr/include -Inanopb/generator -I. --nanopb_out=. message.proto The options can be defined in file, message and field scopes:: @@ -182,35 +208,40 @@ The options can be defined in file, message and field scopes:: } +pb.h +==== +pb_byte_t +--------- +Type used for storing byte-sized data, such as raw binary input and bytes-type fields. :: + typedef uint_least8_t pb_byte_t; - - - - -pb.h -==== +For most platforms this is equivalent to `uint8_t`. Some platforms however do not support +8-bit variables, and on those platforms 16 or 32 bits need to be used for each byte. pb_type_t --------- -Defines the encoder/decoder behaviour that should be used for a field. :: +Type used to store the type of each field, to control the encoder/decoder behaviour. :: - typedef uint8_t pb_type_t; + typedef uint_least8_t pb_type_t; The low-order nibble of the enumeration values defines the function that can be used for encoding and decoding the field data: -==================== ===== ================================================ -LTYPE identifier Value Storage format -==================== ===== ================================================ -PB_LTYPE_VARINT 0x00 Integer. -PB_LTYPE_SVARINT 0x01 Integer, zigzag encoded. -PB_LTYPE_FIXED32 0x02 32-bit integer or floating point. -PB_LTYPE_FIXED64 0x03 64-bit integer or floating point. -PB_LTYPE_BYTES 0x04 Structure with *size_t* field and byte array. -PB_LTYPE_STRING 0x05 Null-terminated string. -PB_LTYPE_SUBMESSAGE 0x06 Submessage structure. -==================== ===== ================================================ +=========================== ===== ================================================ +LTYPE identifier Value Storage format +=========================== ===== ================================================ +PB_LTYPE_VARINT 0x00 Integer. +PB_LTYPE_UVARINT 0x01 Unsigned integer. +PB_LTYPE_SVARINT 0x02 Integer, zigzag encoded. +PB_LTYPE_FIXED32 0x03 32-bit integer or floating point. +PB_LTYPE_FIXED64 0x04 64-bit integer or floating point. +PB_LTYPE_BYTES 0x05 Structure with *size_t* field and byte array. +PB_LTYPE_STRING 0x06 Null-terminated string. +PB_LTYPE_SUBMESSAGE 0x07 Submessage structure. +PB_LTYPE_EXTENSION 0x08 Point to *pb_extension_t*. +PB_LTYPE_FIXED_LENGTH_BYTES 0x09 Inline *pb_byte_t* array of fixed size. +=========================== ===== ================================================ The bits 4-5 define whether the field is required, optional or repeated: @@ -242,14 +273,14 @@ pb_field_t ---------- Describes a single structure field with memory position in relation to others. The descriptions are usually autogenerated. :: - typedef struct _pb_field_t pb_field_t; - struct _pb_field_t { - uint8_t tag; + typedef struct pb_field_s pb_field_t; + struct pb_field_s { + pb_size_t tag; pb_type_t type; - uint8_t data_offset; - int8_t size_offset; - uint8_t data_size; - uint8_t array_size; + pb_size_t data_offset; + pb_ssize_t size_offset; + pb_size_t data_size; + pb_size_t array_size; const void *ptr; } pb_packed; @@ -268,8 +299,8 @@ pb_bytes_array_t An byte array with a field for storing the length:: typedef struct { - size_t size; - uint8_t bytes[1]; + pb_size_t size; + pb_byte_t bytes[1]; } pb_bytes_array_t; In an actual array, the length of *bytes* may be different. @@ -333,12 +364,14 @@ Ties together the extension field type and the storage for the field value:: const pb_extension_type_t *type; void *dest; pb_extension_t *next; + bool found; } pb_extension_t; :type: Pointer to the structure that defines the callback functions. :dest: Pointer to the variable that stores the field value (as used by the default extension callback functions.) :next: Pointer to the next extension handler, or *NULL*. +:found: Decoder sets this to true if the extension was found. PB_GET_ERROR ------------ @@ -382,7 +415,7 @@ pb_ostream_from_buffer ---------------------- Constructs an output stream for writing into a memory buffer. This is just a helper function, it doesn't do anything you couldn't do yourself in a callback function. It uses an internal callback that stores the pointer in stream *state* field. :: - pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); + pb_ostream_t pb_ostream_from_buffer(pb_byte_t *buf, size_t bufsize); :buf: Memory buffer to write into. :bufsize: Maximum number of bytes to write. @@ -394,7 +427,7 @@ pb_write -------- Writes data to an output stream. Always use this function, instead of trying to call stream callback manually. :: - bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); + bool pb_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); :stream: Output stream to write to. :buf: Pointer to buffer with the data to be written. @@ -435,11 +468,22 @@ This function does this, and it is compatible with *parseDelimitedFrom* in Googl Writing packed arrays is a little bit more involved: you need to use `pb_encode_tag` and specify `PB_WT_STRING` as the wire type. Then you need to know exactly how much data you are going to write, and use `pb_encode_varint`_ to write out the number of bytes before writing the actual data. Substreams can be used to determine the number of bytes beforehand; see `pb_encode_submessage`_ source code for an example. +pb_get_encoded_size +------------------- +Calculates the length of the encoded message. :: + + bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *src_struct); + +:size: Calculated size of the encoded message. +:fields: A field description array, usually autogenerated. +:src_struct: Pointer to the data that will be serialized. +:returns: True on success, false on detectable errors in field description or if a field encoder returns false. + pb_encode_tag ------------- Starts a field in the Protocol Buffers binary format: encodes the field number and the wire type of the data. :: - bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, int field_number); + bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number); :stream: Output stream to write to. 1-5 bytes will be written. :wiretype: PB_WT_VARINT, PB_WT_64BIT, PB_WT_STRING or PB_WT_32BIT @@ -460,14 +504,14 @@ This function only considers the LTYPE of the field. You can use it from your fi Wire type mapping is as follows: -========================= ============ -LTYPEs Wire type -========================= ============ -VARINT, SVARINT PB_WT_VARINT -FIXED64 PB_WT_64BIT -STRING, BYTES, SUBMESSAGE PB_WT_STRING -FIXED32 PB_WT_32BIT -========================= ============ +============================================= ============ +LTYPEs Wire type +============================================= ============ +VARINT, UVARINT, SVARINT PB_WT_VARINT +FIXED64 PB_WT_64BIT +STRING, BYTES, SUBMESSAGE, FIXED_LENGTH_BYTES PB_WT_STRING +FIXED32 PB_WT_32BIT +============================================= ============ pb_encode_varint ---------------- @@ -493,7 +537,7 @@ pb_encode_string ---------------- Writes the length of a string as varint and then contents of the string. Works for fields of type `bytes` and `string`:: - bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size); + bool pb_encode_string(pb_ostream_t *stream, const pb_byte_t *buffer, size_t size); :stream: Output stream to write to. :buffer: Pointer to string data. @@ -553,7 +597,7 @@ pb_istream_from_buffer ---------------------- Helper function for creating an input stream that reads data from a memory buffer. :: - pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); + pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize); :buf: Pointer to byte array to read from. :bufsize: Size of the byte array. @@ -563,7 +607,7 @@ pb_read ------- Read data from input stream. Always use this function, don't try to call the stream callback directly. :: - bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); + bool pb_read(pb_istream_t *stream, pb_byte_t *buf, size_t count); :stream: Input stream to read from. :buf: Buffer to store the data to, or NULL to just read data without storing it anywhere. @@ -622,39 +666,21 @@ This function is compatible with *writeDelimitedTo* in the Google's Protocol Buf pb_release ---------- -Releases any dynamically allocated fields. +Releases any dynamically allocated fields:: void pb_release(const pb_field_t fields[], void *dest_struct); :fields: A field description array. Usually autogenerated. -:dest_struct: Pointer to structure where data will be stored. +:dest_struct: Pointer to structure where data is stored. If NULL, function does nothing. This function is only available if *PB_ENABLE_MALLOC* is defined. It will release any pointer type fields in the structure and set the pointers to NULL. -pb_skip_varint --------------- -Skip a varint_ encoded integer without decoding it. :: - - bool pb_skip_varint(pb_istream_t *stream); - -:stream: Input stream to read from. Will read 1 byte at a time until the MSB is clear. -:returns: True on success, false on IO error. - -pb_skip_string --------------- -Skip a varint-length-prefixed string. This means skipping a value with wire type PB_WT_STRING. :: - - bool pb_skip_string(pb_istream_t *stream); - -:stream: Input stream to read from. -:returns: True on success, false on IO error or length exceeding uint32_t. - pb_decode_tag ------------- Decode the tag that comes before field in the protobuf encoding:: - bool pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, int *tag, bool *eof); + bool pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, uint32_t *tag, bool *eof); :stream: Input stream to read from. :wire_type: Pointer to variable where to store the wire type of the field. @@ -721,10 +747,9 @@ pb_decode_fixed64 ----------------- Decode a *fixed64*, *sfixed64* or *double* value. :: - bool pb_dec_fixed(pb_istream_t *stream, const pb_field_t *field, void *dest); + bool pb_decode_fixed64(pb_istream_t *stream, void *dest); :stream: Input stream to read from. 8 bytes will be read. -:field: Not used. :dest: Pointer to destination *int64_t*, *uint64_t* or *double*. :returns: True on success, false on IO errors. diff --git a/docs/security.rst b/docs/security.rst index e865f83..d854612 100644 --- a/docs/security.rst +++ b/docs/security.rst @@ -26,9 +26,9 @@ The following data is regarded as **trusted**. It must be under the control of the application writer. Malicious data in these structures could cause security issues, such as execution of arbitrary code: -1. Callback and extension fields in message structures given to pb_encode() - and pb_decode(). These fields are memory pointers, and are generated - depending on the .proto file. +1. Callback, pointer and extension fields in message structures given to + pb_encode() and pb_decode(). These fields are memory pointers, and are + generated depending on the message definition in the .proto file. 2. The automatically generated field definitions, i.e. *pb_field_t* lists. 3. Contents of the *pb_istream_t* and *pb_ostream_t* structures (this does not mean the contents of the stream itself, just the stream definition). @@ -38,8 +38,11 @@ these will cause "garbage in, garbage out" behaviour. It will not cause buffer overflows, information disclosure or other security problems: 1. All data read from *pb_istream_t*. -2. All fields in message structures, except callbacks and extensions. - (Beginning with nanopb-0.2.4, in earlier versions the field sizes are partially unchecked.) +2. All fields in message structures, except: + + - callbacks (*pb_callback_t* structures) + - pointer fields (malloc support) and *_count* fields for pointers + - extensions (*pb_extension_t* structures) Invariants ========== @@ -76,4 +79,6 @@ The following list is not comprehensive: stop a denial of service attack from using an infinite message. 4. If using network sockets as streams, a timeout should be set to stop denial of service attacks. - +5. If using *malloc()* support, some method of limiting memory use should be + employed. This can be done by defining custom *pb_realloc()* function. + Nanopb will properly detect and handle failed memory allocations. diff --git a/examples/cmake_relpath/CMakeLists.txt b/examples/cmake_relpath/CMakeLists.txt new file mode 100644 index 0000000..e7727d8 --- /dev/null +++ b/examples/cmake_relpath/CMakeLists.txt @@ -0,0 +1,18 @@ +cmake_minimum_required(VERSION 2.8) +project(NANOPB_CMAKE_SIMPLE C) + +set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../extra) +find_package(Nanopb REQUIRED) +include_directories(${NANOPB_INCLUDE_DIRS}) + +nanopb_generate_cpp(PROTO_SRCS PROTO_HDRS RELPATH proto + proto/simple.proto proto/sub/unlucky.proto) + +include_directories(${CMAKE_CURRENT_BINARY_DIR}) +#add_custom_target(generate_proto_sources DEPENDS ${PROTO_SRCS} ${PROTO_HDRS}) +set_source_files_properties(${PROTO_SRCS} ${PROTO_HDRS} + PROPERTIES GENERATED TRUE) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -g -O0") + +add_executable(simple simple.c ${PROTO_SRCS} ${PROTO_HDRS}) diff --git a/examples/cmake_relpath/README.txt b/examples/cmake_relpath/README.txt new file mode 100644 index 0000000..aa0f3f3 --- /dev/null +++ b/examples/cmake_relpath/README.txt @@ -0,0 +1,18 @@ +Nanopb example "simple" using CMake +======================= + +This example is the same as the simple nanopb example but built using CMake. + +Example usage +------------- + +On Linux, create a build directory and then call cmake: + + nanopb/examples/cmake_simple$ mkdir build + nanopb/examples/cmake_simple$ cd build/ + nanopb/examples/cmake_simple/build$ cmake .. + nanopb/examples/cmake_simple/build$ make + +After that, you can run it with the command: ./simple + +On other platforms supported by CMake, refer to CMake instructions. diff --git a/examples/cmake_relpath/proto/simple.proto b/examples/cmake_relpath/proto/simple.proto new file mode 100644 index 0000000..3bf4ad1 --- /dev/null +++ b/examples/cmake_relpath/proto/simple.proto @@ -0,0 +1,11 @@ +// A very simple protocol definition, consisting of only +// one message. +syntax = "proto2"; + +import "sub/unlucky.proto"; + +message SimpleMessage { + required int32 lucky_number = 1; + required UnluckyNumber unlucky = 2; +} + diff --git a/examples/cmake_relpath/proto/sub/unlucky.proto b/examples/cmake_relpath/proto/sub/unlucky.proto new file mode 100644 index 0000000..97a42c9 --- /dev/null +++ b/examples/cmake_relpath/proto/sub/unlucky.proto @@ -0,0 +1,5 @@ +syntax = "proto2"; + +message UnluckyNumber { + required uint32 number = 1; +} diff --git a/examples/cmake_relpath/simple.c b/examples/cmake_relpath/simple.c new file mode 100644 index 0000000..231886c --- /dev/null +++ b/examples/cmake_relpath/simple.c @@ -0,0 +1,73 @@ +#include <stdio.h> +#include <pb_encode.h> +#include <pb_decode.h> +#include "simple.pb.h" + +int main() +{ + /* This is the buffer where we will store our message. */ + uint8_t buffer[128]; + size_t message_length; + bool status; + + /* Encode our message */ + { + /* Allocate space on the stack to store the message data. + * + * Nanopb generates simple struct definitions for all the messages. + * - check out the contents of simple.pb.h! + * It is a good idea to always initialize your structures + * so that you do not have garbage data from RAM in there. + */ + SimpleMessage message = SimpleMessage_init_zero; + + /* Create a stream that will write to our buffer. */ + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Fill in the lucky number */ + message.lucky_number = 13; + message.unlucky.number = 42; + + /* Now we are ready to encode the message! */ + status = pb_encode(&stream, SimpleMessage_fields, &message); + message_length = stream.bytes_written; + + /* Then just check for any errors.. */ + if (!status) + { + printf("Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + } + + /* Now we could transmit the message over network, store it in a file or + * wrap it to a pigeon's leg. + */ + + /* But because we are lazy, we will just decode it immediately. */ + + { + /* Allocate space for the decoded message. */ + SimpleMessage message = SimpleMessage_init_zero; + + /* Create a stream that reads from the buffer. */ + pb_istream_t stream = pb_istream_from_buffer(buffer, message_length); + + /* Now we are ready to decode the message. */ + status = pb_decode(&stream, SimpleMessage_fields, &message); + + /* Check for errors... */ + if (!status) + { + printf("Decoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + + /* Print the data contained in the message. */ + printf("Your lucky number was %d!\n", message.lucky_number); + printf("Your unlucky number was %u!\n", message.unlucky.number); + } + + return 0; +} + diff --git a/examples/cmake_simple/CMakeLists.txt b/examples/cmake_simple/CMakeLists.txt new file mode 100644 index 0000000..e5f33a0 --- /dev/null +++ b/examples/cmake_simple/CMakeLists.txt @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 2.8) +project(NANOPB_CMAKE_SIMPLE C) + +set(CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../extra) +find_package(Nanopb REQUIRED) +include_directories(${NANOPB_INCLUDE_DIRS}) + +nanopb_generate_cpp(PROTO_SRCS PROTO_HDRS simple.proto) +include_directories(${CMAKE_CURRENT_BINARY_DIR}) +#add_custom_target(generate_proto_sources DEPENDS ${PROTO_SRCS} ${PROTO_HDRS}) +set_source_files_properties(${PROTO_SRCS} ${PROTO_HDRS} + PROPERTIES GENERATED TRUE) + +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Werror -g -O0") + +add_executable(simple simple.c ${PROTO_SRCS} ${PROTO_HDRS}) diff --git a/examples/cmake_simple/README.txt b/examples/cmake_simple/README.txt new file mode 100644 index 0000000..aa0f3f3 --- /dev/null +++ b/examples/cmake_simple/README.txt @@ -0,0 +1,18 @@ +Nanopb example "simple" using CMake +======================= + +This example is the same as the simple nanopb example but built using CMake. + +Example usage +------------- + +On Linux, create a build directory and then call cmake: + + nanopb/examples/cmake_simple$ mkdir build + nanopb/examples/cmake_simple$ cd build/ + nanopb/examples/cmake_simple/build$ cmake .. + nanopb/examples/cmake_simple/build$ make + +After that, you can run it with the command: ./simple + +On other platforms supported by CMake, refer to CMake instructions. diff --git a/examples/cmake_simple/simple.c b/examples/cmake_simple/simple.c new file mode 100644 index 0000000..1f6b137 --- /dev/null +++ b/examples/cmake_simple/simple.c @@ -0,0 +1,71 @@ +#include <stdio.h> +#include <pb_encode.h> +#include <pb_decode.h> +#include "simple.pb.h" + +int main() +{ + /* This is the buffer where we will store our message. */ + uint8_t buffer[128]; + size_t message_length; + bool status; + + /* Encode our message */ + { + /* Allocate space on the stack to store the message data. + * + * Nanopb generates simple struct definitions for all the messages. + * - check out the contents of simple.pb.h! + * It is a good idea to always initialize your structures + * so that you do not have garbage data from RAM in there. + */ + SimpleMessage message = SimpleMessage_init_zero; + + /* Create a stream that will write to our buffer. */ + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Fill in the lucky number */ + message.lucky_number = 13; + + /* Now we are ready to encode the message! */ + status = pb_encode(&stream, SimpleMessage_fields, &message); + message_length = stream.bytes_written; + + /* Then just check for any errors.. */ + if (!status) + { + printf("Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + } + + /* Now we could transmit the message over network, store it in a file or + * wrap it to a pigeon's leg. + */ + + /* But because we are lazy, we will just decode it immediately. */ + + { + /* Allocate space for the decoded message. */ + SimpleMessage message = SimpleMessage_init_zero; + + /* Create a stream that reads from the buffer. */ + pb_istream_t stream = pb_istream_from_buffer(buffer, message_length); + + /* Now we are ready to decode the message. */ + status = pb_decode(&stream, SimpleMessage_fields, &message); + + /* Check for errors... */ + if (!status) + { + printf("Decoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + + /* Print the data contained in the message. */ + printf("Your lucky number was %d!\n", message.lucky_number); + } + + return 0; +} + diff --git a/examples/cmake_simple/simple.proto b/examples/cmake_simple/simple.proto new file mode 100644 index 0000000..5c73a3b --- /dev/null +++ b/examples/cmake_simple/simple.proto @@ -0,0 +1,9 @@ +// A very simple protocol definition, consisting of only +// one message. + +syntax = "proto2"; + +message SimpleMessage { + required int32 lucky_number = 1; +} + diff --git a/examples/network_server/client.c b/examples/network_server/client.c index e6e9a2e..ca0c68e 100644 --- a/examples/network_server/client.c +++ b/examples/network_server/client.c @@ -23,9 +23,13 @@ #include "fileproto.pb.h" #include "common.h" +/* This callback function will be called once for each filename received + * from the server. The filenames will be printed out immediately, so that + * no memory has to be allocated for them. + */ bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void **arg) { - FileInfo fileinfo; + FileInfo fileinfo = {}; if (!pb_decode(stream, FileInfo_fields, &fileinfo)) return false; @@ -35,51 +39,66 @@ bool printfile_callback(pb_istream_t *stream, const pb_field_t *field, void **ar return true; } +/* This function sends a request to socket 'fd' to list the files in + * directory given in 'path'. The results received from server will + * be printed to stdout. + */ bool listdir(int fd, char *path) { - ListFilesRequest request; - ListFilesResponse response; - pb_istream_t input = pb_istream_from_socket(fd); - pb_ostream_t output = pb_ostream_from_socket(fd); - uint8_t zero = 0; - - if (path == NULL) - { - request.has_path = false; - } - else + /* Construct and send the request to server */ { - request.has_path = true; - if (strlen(path) + 1 > sizeof(request.path)) + ListFilesRequest request = {}; + pb_ostream_t output = pb_ostream_from_socket(fd); + + /* In our protocol, path is optional. If it is not given, + * the server will list the root directory. */ + if (path == NULL) { - fprintf(stderr, "Too long path.\n"); - return false; + request.has_path = false; + } + else + { + request.has_path = true; + if (strlen(path) + 1 > sizeof(request.path)) + { + fprintf(stderr, "Too long path.\n"); + return false; + } + + strcpy(request.path, path); } - strcpy(request.path, path); - } - - if (!pb_encode(&output, ListFilesRequest_fields, &request)) - { - fprintf(stderr, "Encoding failed.\n"); - return false; - } - - /* We signal the end of request with a 0 tag. */ - pb_write(&output, &zero, 1); - - response.file.funcs.decode = &printfile_callback; - - if (!pb_decode(&input, ListFilesResponse_fields, &response)) - { - fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&input)); - return false; + /* Encode the request. It is written to the socket immediately + * through our custom stream. */ + if (!pb_encode_delimited(&output, ListFilesRequest_fields, &request)) + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&output)); + return false; + } } - if (response.path_error) + /* Read back the response from server */ { - fprintf(stderr, "Server reported error.\n"); - return false; + ListFilesResponse response = {}; + pb_istream_t input = pb_istream_from_socket(fd); + + /* Give a pointer to our callback function, which will handle the + * filenames as they arrive. */ + response.file.funcs.decode = &printfile_callback; + + if (!pb_decode_delimited(&input, ListFilesResponse_fields, &response)) + { + fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&input)); + return false; + } + + /* If the message from server decodes properly, but directory was + * not found on server side, we get path_error == true. */ + if (response.path_error) + { + fprintf(stderr, "Server reported error.\n"); + return false; + } } return true; @@ -96,6 +115,7 @@ int main(int argc, char **argv) sockfd = socket(AF_INET, SOCK_STREAM, 0); + /* Connect to server running on localhost:1234 */ memset(&servaddr, 0, sizeof(servaddr)); servaddr.sin_family = AF_INET; servaddr.sin_addr.s_addr = htonl(INADDR_LOOPBACK); @@ -107,9 +127,11 @@ int main(int argc, char **argv) return 1; } + /* Send the directory listing request */ if (!listdir(sockfd, path)) return 2; + /* Close connection */ close(sockfd); return 0; diff --git a/examples/network_server/fileproto.proto b/examples/network_server/fileproto.proto index 3e70c49..5640b8d 100644 --- a/examples/network_server/fileproto.proto +++ b/examples/network_server/fileproto.proto @@ -2,6 +2,8 @@ // // See also the nanopb-specific options in fileproto.options. +syntax = "proto2"; + message ListFilesRequest { optional string path = 1 [default = "/"]; } diff --git a/examples/network_server/server.c b/examples/network_server/server.c index 9a9c264..f500dcd 100644 --- a/examples/network_server/server.c +++ b/examples/network_server/server.c @@ -23,11 +23,16 @@ #include "fileproto.pb.h" #include "common.h" +/* This callback function will be called once during the encoding. + * It will write out any number of FileInfo entries, without consuming unnecessary memory. + * This is accomplished by fetching the filenames one at a time and encoding them + * immediately. + */ bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) { DIR *dir = (DIR*) *arg; struct dirent *file; - FileInfo fileinfo; + FileInfo fileinfo = {}; while ((file = readdir(dir)) != NULL) { @@ -35,53 +40,76 @@ bool listdir_callback(pb_ostream_t *stream, const pb_field_t *field, void * cons strncpy(fileinfo.name, file->d_name, sizeof(fileinfo.name)); fileinfo.name[sizeof(fileinfo.name) - 1] = '\0'; + /* This encodes the header for the field, based on the constant info + * from pb_field_t. */ if (!pb_encode_tag_for_field(stream, field)) return false; + /* This encodes the data for the field, based on our FileInfo structure. */ if (!pb_encode_submessage(stream, FileInfo_fields, &fileinfo)) return false; } + /* Because the main program uses pb_encode_delimited(), this callback will be + * called twice. Rewind the directory for the next call. */ + rewinddir(dir); + return true; } +/* Handle one arriving client connection. + * Clients are expected to send a ListFilesRequest, terminated by a '0'. + * Server will respond with a ListFilesResponse message. + */ void handle_connection(int connfd) { - ListFilesRequest request; - ListFilesResponse response; - pb_istream_t input = pb_istream_from_socket(connfd); - pb_ostream_t output = pb_ostream_from_socket(connfd); - DIR *directory; + DIR *directory = NULL; - if (!pb_decode(&input, ListFilesRequest_fields, &request)) + /* Decode the message from the client and open the requested directory. */ { - printf("Decode failed: %s\n", PB_GET_ERROR(&input)); - return; + ListFilesRequest request = {}; + pb_istream_t input = pb_istream_from_socket(connfd); + + if (!pb_decode_delimited(&input, ListFilesRequest_fields, &request)) + { + printf("Decode failed: %s\n", PB_GET_ERROR(&input)); + return; + } + + directory = opendir(request.path); + printf("Listing directory: %s\n", request.path); } - directory = opendir(request.path); - - printf("Listing directory: %s\n", request.path); - - if (directory == NULL) + /* List the files in the directory and transmit the response to client */ { - perror("opendir"); + ListFilesResponse response = {}; + pb_ostream_t output = pb_ostream_from_socket(connfd); - response.has_path_error = true; - response.path_error = true; - response.file.funcs.encode = NULL; - } - else - { - response.has_path_error = false; - response.file.funcs.encode = &listdir_callback; - response.file.arg = directory; + if (directory == NULL) + { + perror("opendir"); + + /* Directory was not found, transmit error status */ + response.has_path_error = true; + response.path_error = true; + response.file.funcs.encode = NULL; + } + else + { + /* Directory was found, transmit filenames */ + response.has_path_error = false; + response.file.funcs.encode = &listdir_callback; + response.file.arg = directory; + } + + if (!pb_encode_delimited(&output, ListFilesResponse_fields, &response)) + { + printf("Encoding failed: %s\n", PB_GET_ERROR(&output)); + } } - if (!pb_encode(&output, ListFilesResponse_fields, &response)) - { - printf("Encoding failed.\n"); - } + if (directory != NULL) + closedir(directory); } int main(int argc, char **argv) @@ -90,8 +118,8 @@ int main(int argc, char **argv) struct sockaddr_in servaddr; int reuse = 1; + /* Listen on localhost:1234 for TCP connections */ listenfd = socket(AF_INET, SOCK_STREAM, 0); - setsockopt(listenfd, SOL_SOCKET, SO_REUSEADDR, &reuse, sizeof(reuse)); memset(&servaddr, 0, sizeof(servaddr)); @@ -112,6 +140,7 @@ int main(int argc, char **argv) for(;;) { + /* Wait for a client */ connfd = accept(listenfd, NULL, NULL); if (connfd < 0) @@ -128,4 +157,6 @@ int main(int argc, char **argv) close(connfd); } + + return 0; } diff --git a/examples/simple/Makefile b/examples/simple/Makefile index 02a4c3f..970a865 100644 --- a/examples/simple/Makefile +++ b/examples/simple/Makefile @@ -10,6 +10,7 @@ CSRC = simple.c # The main program CSRC += simple.pb.c # The compiled protocol definition CSRC += $(NANOPB_DIR)/pb_encode.c # The nanopb encoder CSRC += $(NANOPB_DIR)/pb_decode.c # The nanopb decoder +CSRC += $(NANOPB_DIR)/pb_common.c # The nanopb common parts # Build rule for the main program simple: $(CSRC) diff --git a/examples/simple/rules.mk b/examples/simple/rules.mk new file mode 100644 index 0000000..1e4221d --- /dev/null +++ b/examples/simple/rules.mk @@ -0,0 +1,24 @@ +NANOPB_DIR := external/nanopb-0.3.9.1-linux-x86 + +LOCAL_DIR := $(GET_LOCAL_DIR) + +MODULE := $(LOCAL_DIR) + +MODULE_SRCS += \ + $(LOCAL_DIR)/simple.c \ + $(LOCAL_DIR)/simple.pb.c \ + $(NANOPB_DIR)/pb_encode.c \ + $(NANOPB_DIR)/pb_decode.c \ + $(NANOPB_DIR)/pb_common.c + +MODULE_CPPFLAGS += -std=c++11 + +MODULE_INCLUDES += \ + $(NANOPB_DIR) + +$(LOCAL_DIR)/simple.pb.c: $(LOCAL_DIR)/simple.proto + $(PROTOC) $(PROTOC_OPTS) --nanopb_out=$(LOCAL_DIR) $(LOCAL_DIR)/simple.proto + +include $(NANOPB_DIR)/extra/nanopb.mk +include make/module.mk + diff --git a/examples/simple/simple.c b/examples/simple/simple.c index 3127230..c16ec52 100644 --- a/examples/simple/simple.c +++ b/examples/simple/simple.c @@ -15,8 +15,11 @@ int main() /* Allocate space on the stack to store the message data. * * Nanopb generates simple struct definitions for all the messages. - * - check out the contents of simple.pb.h! */ - SimpleMessage message; + * - check out the contents of simple.pb.h! + * It is a good idea to always initialize your structures + * so that you do not have garbage data from RAM in there. + */ + SimpleMessage message = SimpleMessage_init_zero; /* Create a stream that will write to our buffer. */ pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); @@ -44,7 +47,7 @@ int main() { /* Allocate space for the decoded message. */ - SimpleMessage message; + SimpleMessage message = SimpleMessage_init_zero; /* Create a stream that reads from the buffer. */ pb_istream_t stream = pb_istream_from_buffer(buffer, message_length); @@ -60,7 +63,7 @@ int main() } /* Print the data contained in the message. */ - printf("Your lucky number was %d!\n", message.lucky_number); + printf("Your lucky number was %d!\n", (int)message.lucky_number); } return 0; diff --git a/examples/simple/simple.pb.c b/examples/simple/simple.pb.c new file mode 100644 index 0000000..481aa7d --- /dev/null +++ b/examples/simple/simple.pb.c @@ -0,0 +1,19 @@ +/* Automatically generated nanopb constant definitions */ +/* Generated by nanopb-0.3.9.1 at Fri May 25 16:28:54 2018. */ + +#include "simple.pb.h" + +/* @@protoc_insertion_point(includes) */ +#if PB_PROTO_HEADER_VERSION != 30 +#error Regenerate this file with the current version of nanopb generator. +#endif + + + +const pb_field_t SimpleMessage_fields[2] = { + PB_FIELD( 1, INT32 , REQUIRED, STATIC , FIRST, SimpleMessage, lucky_number, lucky_number, 0), + PB_LAST_FIELD +}; + + +/* @@protoc_insertion_point(eof) */ diff --git a/examples/simple/simple.pb.h b/examples/simple/simple.pb.h new file mode 100644 index 0000000..f4614db --- /dev/null +++ b/examples/simple/simple.pb.h @@ -0,0 +1,51 @@ +/* Automatically generated nanopb header */ +/* Generated by nanopb-0.3.9.1 at Fri May 25 16:28:54 2018. */ + +#ifndef PB_SIMPLE_PB_H_INCLUDED +#define PB_SIMPLE_PB_H_INCLUDED +#include <pb.h> + +/* @@protoc_insertion_point(includes) */ +#if PB_PROTO_HEADER_VERSION != 30 +#error Regenerate this file with the current version of nanopb generator. +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Struct definitions */ +typedef struct _SimpleMessage { + int32_t lucky_number; +/* @@protoc_insertion_point(struct:SimpleMessage) */ +} SimpleMessage; + +/* Default values for struct fields */ + +/* Initializer values for message structs */ +#define SimpleMessage_init_default {0} +#define SimpleMessage_init_zero {0} + +/* Field tags (for use in manual encoding/decoding) */ +#define SimpleMessage_lucky_number_tag 1 + +/* Struct field encoding specification for nanopb */ +extern const pb_field_t SimpleMessage_fields[2]; + +/* Maximum encoded size of messages (where known) */ +#define SimpleMessage_size 11 + +/* Message IDs (where set with "msgid" option) */ +#ifdef PB_MSGID + +#define SIMPLE_MESSAGES \ + + +#endif + +#ifdef __cplusplus +} /* extern "C" */ +#endif +/* @@protoc_insertion_point(eof) */ + +#endif diff --git a/examples/simple/simple.proto b/examples/simple/simple.proto index 26e72f4..5c73a3b 100644 --- a/examples/simple/simple.proto +++ b/examples/simple/simple.proto @@ -1,6 +1,8 @@ // A very simple protocol definition, consisting of only // one message. +syntax = "proto2"; + message SimpleMessage { required int32 lucky_number = 1; } diff --git a/examples/using_double_on_avr/doubleproto.proto b/examples/using_double_on_avr/doubleproto.proto index d8b7f2d..72d3f9c 100644 --- a/examples/using_double_on_avr/doubleproto.proto +++ b/examples/using_double_on_avr/doubleproto.proto @@ -1,4 +1,6 @@ // A message containing doubles, as used by other applications. +syntax = "proto2"; + message DoubleMessage { required double field1 = 1; required double field2 = 2; diff --git a/examples/using_union_messages/README.txt b/examples/using_union_messages/README.txt index 7a1e75d..d4b7fd2 100644 --- a/examples/using_union_messages/README.txt +++ b/examples/using_union_messages/README.txt @@ -17,6 +17,9 @@ we actually want. Similarly when decoding, we can manually read the tag of the top level message, and only then allocate the memory for the submessage after we already know its type. +NOTE: There is a newer protobuf feature called `oneof` that is also supported +by nanopb. It might be a better option for new code. + Example usage ------------- diff --git a/examples/using_union_messages/unionproto.proto b/examples/using_union_messages/unionproto.proto index d7c9de2..209df0d 100644 --- a/examples/using_union_messages/unionproto.proto +++ b/examples/using_union_messages/unionproto.proto @@ -5,6 +5,8 @@ // but they are commonly implemented by filling out exactly one of // several optional fields. +syntax = "proto2"; + message MsgType1 { required int32 value = 1; diff --git a/extra/FindNanopb.cmake b/extra/FindNanopb.cmake index c804e70..bba341d 100644 --- a/extra/FindNanopb.cmake +++ b/extra/FindNanopb.cmake @@ -1,10 +1,6 @@ # This is an example script for use with CMake projects for locating and configuring # the nanopb library. # -# The following varialbes have to be set: -# -# NANOPB_SRC_ROOT_FOLDER - Path to nanopb source folder -# # The following variables can be set and are optional: # # @@ -16,6 +12,13 @@ # NANOPB_IMPORT_DIRS - List of additional directories to be searched for # imported .proto files. # +# NANOPB_OPTIONS - List of options passed to nanopb. +# +# NANOPB_DEPENDS - List of files to be used as dependencies +# for the generated source and header files. These +# files are not directly passed as options to +# nanopb but rather their directories. +# # NANOPB_GENERATE_CPP_APPEND_PATH - By default -I will be passed to protoc # for each directory where a proto file is referenced. # Set to FALSE if you want to disable this behaviour. @@ -26,23 +29,28 @@ # NANOPB_INCLUDE_DIRS - Include directories for Google Protocol Buffers # # The following cache variables are also available to set or use: -# NANOPB_GENERATOR_EXECUTABLE - The nanopb generator # PROTOBUF_PROTOC_EXECUTABLE - The protoc compiler +# NANOPB_GENERATOR_SOURCE_DIR - The nanopb generator source # # ==================================================================== # # NANOPB_GENERATE_CPP (public function) -# SRCS = Variable to define with autogenerated -# source files -# HDRS = Variable to define with autogenerated -# header files -# ARGN = proto files +# NANOPB_GENERATE_CPP(SRCS HDRS [RELPATH <root-path-of-proto-files>] +# <proto-files>...) +# SRCS = Variable to define with autogenerated source files +# HDRS = Variable to define with autogenerated header files +# If you want to use relative paths in your import statements use the RELPATH +# option. The argument to RELPATH should be the directory that all the +# imports will be relative to. +# When RELPATH is not specified then all proto files can be imported without +# a path. +# # # ==================================================================== # Example: # # set(NANOPB_SRC_ROOT_FOLDER "/path/to/nanopb") -# set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${NANOPB_SRC_ROOT_FOLDER}/cmake) +# set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${NANOPB_SRC_ROOT_FOLDER}/extra) # find_package( Nanopb REQUIRED ) # include_directories(${NANOPB_INCLUDE_DIRS}) # @@ -51,6 +59,19 @@ # include_directories(${CMAKE_CURRENT_BINARY_DIR}) # add_executable(bar bar.cc ${PROTO_SRCS} ${PROTO_HDRS}) # +# Example with RELPATH: +# Assume we have a layout like: +# .../CMakeLists.txt +# .../bar.cc +# .../proto/ +# .../proto/foo.proto (Which contains: import "sub/bar.proto"; ) +# .../proto/sub/bar.proto +# Everything would be the same as the previous example, but the call to +# NANOPB_GENERATE_CPP would change to: +# +# NANOPB_GENERATE_CPP(PROTO_SRCS PROTO_HDRS RELPATH proto +# proto/foo.proto proto/sub/bar.proto) +# # ==================================================================== #============================================================================= @@ -96,62 +117,148 @@ function(NANOPB_GENERATE_CPP SRCS HDRS) - if(NOT ARGN) + cmake_parse_arguments(NANOPB_GENERATE_CPP "" "RELPATH" "" ${ARGN}) + if(NOT NANOPB_GENERATE_CPP_UNPARSED_ARGUMENTS) return() endif() if(NANOPB_GENERATE_CPP_APPEND_PATH) # Create an include path for each file specified - foreach(FIL ${ARGN}) + foreach(FIL ${NANOPB_GENERATE_CPP_UNPARSED_ARGUMENTS}) get_filename_component(ABS_FIL ${FIL} ABSOLUTE) get_filename_component(ABS_PATH ${ABS_FIL} PATH) - - list(FIND _nanobp_include_path ${ABS_PATH} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND _nanobp_include_path -I ${ABS_PATH}) - endif() + list(APPEND _nanopb_include_path "-I${ABS_PATH}") endforeach() else() - set(_nanobp_include_path -I ${CMAKE_CURRENT_SOURCE_DIR}) + set(_nanopb_include_path "-I${CMAKE_CURRENT_SOURCE_DIR}") + endif() + + if(NANOPB_GENERATE_CPP_RELPATH) + list(APPEND _nanopb_include_path "-I${NANOPB_GENERATE_CPP_RELPATH}") endif() if(DEFINED NANOPB_IMPORT_DIRS) foreach(DIR ${NANOPB_IMPORT_DIRS}) get_filename_component(ABS_PATH ${DIR} ABSOLUTE) - list(FIND _nanobp_include_path ${ABS_PATH} _contains_already) - if(${_contains_already} EQUAL -1) - list(APPEND _nanobp_include_path -I ${ABS_PATH}) - endif() + list(APPEND _nanopb_include_path "-I${ABS_PATH}") endforeach() endif() - set(${SRCS}) - set(${HDRS}) - get_filename_component(GENERATOR_PATH ${NANOPB_GENERATOR_EXECUTABLE} PATH) + list(REMOVE_DUPLICATES _nanopb_include_path) + + set(GENERATOR_PATH ${CMAKE_BINARY_DIR}/nanopb/generator) + + set(NANOPB_GENERATOR_EXECUTABLE ${GENERATOR_PATH}/nanopb_generator.py) + set(NANOPB_GENERATOR_PLUGIN ${GENERATOR_PATH}/protoc-gen-nanopb) + + set(GENERATOR_CORE_DIR ${GENERATOR_PATH}/proto) + set(GENERATOR_CORE_SRC + ${GENERATOR_CORE_DIR}/nanopb.proto + ${GENERATOR_CORE_DIR}/plugin.proto) + + # Treat the source diretory as immutable. + # + # Copy the generator directory to the build directory before + # compiling python and proto files. Fixes issues when using the + # same build directory with different python/protobuf versions + # as the binary build directory is discarded across builds. + # + add_custom_command( + OUTPUT ${NANOPB_GENERATOR_EXECUTABLE} ${GENERATOR_CORE_SRC} + COMMAND ${CMAKE_COMMAND} -E copy_directory + ARGS ${NANOPB_GENERATOR_SOURCE_DIR} ${GENERATOR_PATH} + VERBATIM) + + set(GENERATOR_CORE_PYTHON_SRC) + foreach(FIL ${GENERATOR_CORE_SRC}) + get_filename_component(ABS_FIL ${FIL} ABSOLUTE) + get_filename_component(FIL_WE ${FIL} NAME_WE) - foreach(FIL ${ARGN}) + set(output "${GENERATOR_CORE_DIR}/${FIL_WE}_pb2.py") + set(GENERATOR_CORE_PYTHON_SRC ${GENERATOR_CORE_PYTHON_SRC} ${output}) + add_custom_command( + OUTPUT ${output} + COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} + ARGS -I${GENERATOR_PATH}/proto + --python_out=${GENERATOR_CORE_DIR} ${ABS_FIL} + DEPENDS ${ABS_FIL} + VERBATIM) + endforeach() + + if(NANOPB_GENERATE_CPP_RELPATH) + get_filename_component(ABS_ROOT ${NANOPB_GENERATE_CPP_RELPATH} ABSOLUTE) + endif() + foreach(FIL ${NANOPB_GENERATE_CPP_UNPARSED_ARGUMENTS}) get_filename_component(ABS_FIL ${FIL} ABSOLUTE) get_filename_component(FIL_WE ${FIL} NAME_WE) + get_filename_component(FIL_DIR ${FIL} PATH) + set(FIL_PATH_REL) + if(ABS_ROOT) + # Check that the file is under the given "RELPATH" + string(FIND ${ABS_FIL} ${ABS_ROOT} LOC) + if (${LOC} EQUAL 0) + string(REPLACE "${ABS_ROOT}/" "" FIL_REL ${ABS_FIL}) + get_filename_component(FIL_PATH_REL ${FIL_REL} PATH) + file(MAKE_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/${FIL_PATH_REL}) + endif() + endif() + if(NOT FIL_PATH_REL) + set(FIL_PATH_REL ".") + endif() + + list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_PATH_REL}/${FIL_WE}.pb.c") + list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_PATH_REL}/${FIL_WE}.pb.h") + + set(NANOPB_PLUGIN_OPTIONS) + set(NANOPB_OPTIONS_DIRS) + + # If there an options file in the same working directory, set it as a dependency + set(NANOPB_OPTIONS_FILE ${FIL_DIR}/${FIL_WE}.options) + if(EXISTS ${NANOPB_OPTIONS_FILE}) + # Get directory as lookups for dependency options fail if an options + # file is used. The options is still set as a dependency of the + # generated source and header. + get_filename_component(options_dir ${NANOPB_OPTIONS_FILE} DIRECTORY) + list(APPEND NANOPB_OPTIONS_DIRS ${options_dir}) + else() + set(NANOPB_OPTIONS_FILE) + endif() + + # If the dependencies are options files, we need to pass the directories + # as arguments to nanopb + foreach(depends_file ${NANOPB_DEPENDS}) + get_filename_component(ext ${depends_file} EXT) + if(ext STREQUAL ".options") + get_filename_component(depends_dir ${depends_file} DIRECTORY) + list(APPEND NANOPB_OPTIONS_DIRS ${depends_dir}) + endif() + endforeach() - list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c") - list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h") + if(NANOPB_OPTIONS_DIRS) + list(REMOVE_DUPLICATES NANOPB_OPTIONS_DIRS) + endif() + + foreach(options_path ${NANOPB_OPTIONS_DIRS}) + set(NANOPB_PLUGIN_OPTIONS "${NANOPB_PLUGIN_OPTIONS} -I${options_path}") + endforeach() + + if(NANOPB_OPTIONS) + set(NANOPB_PLUGIN_OPTIONS "${NANOPB_PLUGIN_OPTIONS} ${NANOPB_OPTIONS}") + endif() add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb" + OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_PATH_REL}/${FIL_WE}.pb.c" + "${CMAKE_CURRENT_BINARY_DIR}/${FIL_PATH_REL}/${FIL_WE}.pb.h" COMMAND ${PROTOBUF_PROTOC_EXECUTABLE} - ARGS -I${GENERATOR_PATH} -I${CMAKE_CURRENT_BINARY_DIR} ${_nanobp_include_path} -o${FIL_WE}.pb ${ABS_FIL} - DEPENDS ${ABS_FIL} - COMMENT "Running C++ protocol buffer compiler on ${FIL}" + ARGS -I${GENERATOR_PATH} -I${GENERATOR_CORE_DIR} + -I${CMAKE_CURRENT_BINARY_DIR} ${_nanopb_include_path} + --plugin=protoc-gen-nanopb=${NANOPB_GENERATOR_PLUGIN} + "--nanopb_out=${NANOPB_PLUGIN_OPTIONS}:${CMAKE_CURRENT_BINARY_DIR}" ${ABS_FIL} + DEPENDS ${ABS_FIL} ${GENERATOR_CORE_PYTHON_SRC} + ${NANOPB_OPTIONS_FILE} ${NANOPB_DEPENDS} + COMMENT "Running C++ protocol buffer compiler using nanopb plugin on ${FIL}" VERBATIM ) - add_custom_command( - OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.c" - "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb.h" - COMMAND python - ARGS ${NANOPB_GENERATOR_EXECUTABLE} ${FIL_WE}.pb - DEPENDS "${CMAKE_CURRENT_BINARY_DIR}/${FIL_WE}.pb" - COMMENT "Running nanopb generator on ${FIL_WE}.pb" - VERBATIM ) endforeach() set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE) @@ -172,6 +279,12 @@ if(NOT DEFINED NANOPB_GENERATE_CPP_APPEND_PATH) set(NANOPB_GENERATE_CPP_APPEND_PATH TRUE) endif() +# Make a really good guess regarding location of NANOPB_SRC_ROOT_FOLDER +if(NOT DEFINED NANOPB_SRC_ROOT_FOLDER) + get_filename_component(NANOPB_SRC_ROOT_FOLDER + ${CMAKE_CURRENT_LIST_DIR}/.. ABSOLUTE) +endif() + # Find the include directory find_path(NANOPB_INCLUDE_DIRS pb.h @@ -182,8 +295,8 @@ mark_as_advanced(NANOPB_INCLUDE_DIRS) # Find nanopb source files set(NANOPB_SRCS) set(NANOPB_HDRS) -list(APPEND _nanopb_srcs pb_decode.c pb_encode.c) -list(APPEND _nanopb_hdrs pb_decode.h pb_encode.h pb.h) +list(APPEND _nanopb_srcs pb_decode.c pb_encode.c pb_common.c) +list(APPEND _nanopb_hdrs pb_decode.h pb_encode.h pb_common.h pb.h) foreach(FIL ${_nanopb_srcs}) find_file(${FIL}__nano_pb_file NAMES ${FIL} PATHS ${NANOPB_SRC_ROOT_FOLDER} ${NANOPB_INCLUDE_DIRS}) @@ -207,19 +320,21 @@ find_program(PROTOBUF_PROTOC_EXECUTABLE ) mark_as_advanced(PROTOBUF_PROTOC_EXECUTABLE) -# Find nanopb generator -find_file(NANOPB_GENERATOR_EXECUTABLE +# Find nanopb generator source dir +find_path(NANOPB_GENERATOR_SOURCE_DIR NAMES nanopb_generator.py - DOC "nanopb generator" + DOC "nanopb generator source" PATHS ${NANOPB_SRC_ROOT_FOLDER}/generator ) -mark_as_advanced(NANOPB_GENERATOR_EXECUTABLE) +mark_as_advanced(NANOPB_GENERATOR_SOURCE_DIR) + +find_package(PythonInterp REQUIRED) include(FindPackageHandleStandardArgs) FIND_PACKAGE_HANDLE_STANDARD_ARGS(NANOPB DEFAULT_MSG NANOPB_INCLUDE_DIRS NANOPB_SRCS NANOPB_HDRS - NANOPB_GENERATOR_EXECUTABLE + NANOPB_GENERATOR_SOURCE_DIR PROTOBUF_PROTOC_EXECUTABLE ) diff --git a/extra/nanopb-config-version.cmake.in b/extra/nanopb-config-version.cmake.in new file mode 100644 index 0000000..1b51027 --- /dev/null +++ b/extra/nanopb-config-version.cmake.in @@ -0,0 +1,11 @@ +set(PACKAGE_VERSION "@nanopb_VERSION@") + +# Check whether the requested PACKAGE_FIND_VERSION is compatible +if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_COMPATIBLE FALSE) +else() + set(PACKAGE_VERSION_COMPATIBLE TRUE) + if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") + set(PACKAGE_VERSION_EXACT TRUE) + endif() +endif() diff --git a/extra/nanopb-config.cmake b/extra/nanopb-config.cmake new file mode 100644 index 0000000..42d977a --- /dev/null +++ b/extra/nanopb-config.cmake @@ -0,0 +1 @@ +include(${CMAKE_CURRENT_LIST_DIR}/nanopb-targets.cmake) diff --git a/extra/nanopb.mk b/extra/nanopb.mk index 7576bae..5c2cff5 100644 --- a/extra/nanopb.mk +++ b/extra/nanopb.mk @@ -5,7 +5,7 @@ NANOPB_DIR := $(abspath $(dir $(lastword $(MAKEFILE_LIST)))../) # Files for the nanopb core -NANOPB_CORE = $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_decode.c +NANOPB_CORE = $(NANOPB_DIR)/pb_encode.c $(NANOPB_DIR)/pb_decode.c $(NANOPB_DIR)/pb_common.c # Check if we are running on Windows ifdef windir diff --git a/extra/pb_syshdr.h b/extra/pb_syshdr.h index 1ff4823..55d06a3 100644 --- a/extra/pb_syshdr.h +++ b/extra/pb_syshdr.h @@ -24,6 +24,14 @@ typedef signed int int32_t; typedef unsigned int uint32_t; typedef signed long long int64_t; typedef unsigned long long uint64_t; + +/* These are ok for most platforms, unless uint8_t is actually not available, + * in which case you should give the smallest available type. */ +typedef int8_t int_least8_t; +typedef uint8_t uint_least8_t; +typedef uint8_t uint_fast8_t; +typedef int16_t int_least16_t; +typedef uint16_t uint_least16_t; #endif /* stddef.h subset */ diff --git a/generator/Android.bp b/generator/Android.bp new file mode 100644 index 0000000..aa94f65 --- /dev/null +++ b/generator/Android.bp @@ -0,0 +1,23 @@ +python_binary_host { + name: "protoc-gen-nanopb", + main: "nanopb_generator.py", + srcs: [ + "nanopb_generator.py", + "proto/nanopb.proto", + "proto/plugin.proto", + ], + proto: { + canonical_path_from_root: false, + include_dirs: ["external/protobuf/src"], + }, + libs: ["libprotobuf-python"], + version: { + py2: { + enabled: true, + embedded_launcher: true, + }, + py3: { + enabled: false, + }, + }, +} diff --git a/generator/google/__init__.py b/generator/google/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/generator/google/__init__.py +++ /dev/null diff --git a/generator/google/protobuf/__init__.py b/generator/google/protobuf/__init__.py index e69de29..6210a40 100644 --- a/generator/google/protobuf/__init__.py +++ b/generator/google/protobuf/__init__.py @@ -0,0 +1,39 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Copyright 2007 Google Inc. All Rights Reserved. + +__version__ = '3.0.0' + +if __name__ != '__main__': + try: + __import__('pkg_resources').declare_namespace(__name__) + except ImportError: + __path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/generator/google/protobuf/compiler/__init__.py b/generator/google/protobuf/compiler/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/generator/google/protobuf/compiler/__init__.py +++ /dev/null diff --git a/generator/google/protobuf/compiler/plugin_pb2.py b/generator/google/protobuf/compiler/plugin_pb2.py deleted file mode 100644 index 77cc07a..0000000 --- a/generator/google/protobuf/compiler/plugin_pb2.py +++ /dev/null @@ -1,166 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/protobuf/compiler/plugin.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - - -import google.protobuf.descriptor_pb2 - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/protobuf/compiler/plugin.proto', - package='google.protobuf.compiler', - serialized_pb='\n%google/protobuf/compiler/plugin.proto\x12\x18google.protobuf.compiler\x1a google/protobuf/descriptor.proto\"}\n\x14\x43odeGeneratorRequest\x12\x18\n\x10\x66ile_to_generate\x18\x01 \x03(\t\x12\x11\n\tparameter\x18\x02 \x01(\t\x12\x38\n\nproto_file\x18\x0f \x03(\x0b\x32$.google.protobuf.FileDescriptorProto\"\xaa\x01\n\x15\x43odeGeneratorResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\x42\n\x04\x66ile\x18\x0f \x03(\x0b\x32\x34.google.protobuf.compiler.CodeGeneratorResponse.File\x1a>\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x17\n\x0finsertion_point\x18\x02 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x0f \x01(\tB,\n\x1c\x63om.google.protobuf.compilerB\x0cPluginProtos') - - - - -_CODEGENERATORREQUEST = _descriptor.Descriptor( - name='CodeGeneratorRequest', - full_name='google.protobuf.compiler.CodeGeneratorRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='file_to_generate', full_name='google.protobuf.compiler.CodeGeneratorRequest.file_to_generate', index=0, - number=1, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='parameter', full_name='google.protobuf.compiler.CodeGeneratorRequest.parameter', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='proto_file', full_name='google.protobuf.compiler.CodeGeneratorRequest.proto_file', index=2, - number=15, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=101, - serialized_end=226, -) - - -_CODEGENERATORRESPONSE_FILE = _descriptor.Descriptor( - name='File', - full_name='google.protobuf.compiler.CodeGeneratorResponse.File', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='insertion_point', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.insertion_point', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='content', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.content', index=2, - number=15, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=337, - serialized_end=399, -) - -_CODEGENERATORRESPONSE = _descriptor.Descriptor( - name='CodeGeneratorResponse', - full_name='google.protobuf.compiler.CodeGeneratorResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='error', full_name='google.protobuf.compiler.CodeGeneratorResponse.error', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='file', full_name='google.protobuf.compiler.CodeGeneratorResponse.file', index=1, - number=15, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_CODEGENERATORRESPONSE_FILE, ], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=229, - serialized_end=399, -) - -_CODEGENERATORREQUEST.fields_by_name['proto_file'].message_type = google.protobuf.descriptor_pb2._FILEDESCRIPTORPROTO -_CODEGENERATORRESPONSE_FILE.containing_type = _CODEGENERATORRESPONSE; -_CODEGENERATORRESPONSE.fields_by_name['file'].message_type = _CODEGENERATORRESPONSE_FILE -DESCRIPTOR.message_types_by_name['CodeGeneratorRequest'] = _CODEGENERATORREQUEST -DESCRIPTOR.message_types_by_name['CodeGeneratorResponse'] = _CODEGENERATORRESPONSE - -class CodeGeneratorRequest(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _CODEGENERATORREQUEST - - # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorRequest) - -class CodeGeneratorResponse(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - - class File(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _CODEGENERATORRESPONSE_FILE - - # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse.File) - DESCRIPTOR = _CODEGENERATORRESPONSE - - # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse) - - -DESCRIPTOR.has_options = True -DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), '\n\034com.google.protobuf.compilerB\014PluginProtos') -# @@protoc_insertion_point(module_scope) diff --git a/generator/google/protobuf/descriptor.py b/generator/google/protobuf/descriptor.py deleted file mode 100644 index eb13eda..0000000 --- a/generator/google/protobuf/descriptor.py +++ /dev/null @@ -1,713 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Descriptors essentially contain exactly the information found in a .proto -file, in types that make this information accessible in Python. -""" - -__author__ = 'robinson@google.com (Will Robinson)' - - -from google.protobuf.internal import api_implementation - - -if api_implementation.Type() == 'cpp': - if api_implementation.Version() == 2: - from google.protobuf.internal.cpp import _message - else: - from google.protobuf.internal import cpp_message - - -class Error(Exception): - """Base error for this module.""" - - -class TypeTransformationError(Error): - """Error transforming between python proto type and corresponding C++ type.""" - - -class DescriptorBase(object): - - """Descriptors base class. - - This class is the base of all descriptor classes. It provides common options - related functionaility. - - Attributes: - has_options: True if the descriptor has non-default options. Usually it - is not necessary to read this -- just call GetOptions() which will - happily return the default instance. However, it's sometimes useful - for efficiency, and also useful inside the protobuf implementation to - avoid some bootstrapping issues. - """ - - def __init__(self, options, options_class_name): - """Initialize the descriptor given its options message and the name of the - class of the options message. The name of the class is required in case - the options message is None and has to be created. - """ - self._options = options - self._options_class_name = options_class_name - - # Does this descriptor have non-default options? - self.has_options = options is not None - - def _SetOptions(self, options, options_class_name): - """Sets the descriptor's options - - This function is used in generated proto2 files to update descriptor - options. It must not be used outside proto2. - """ - self._options = options - self._options_class_name = options_class_name - - # Does this descriptor have non-default options? - self.has_options = options is not None - - def GetOptions(self): - """Retrieves descriptor options. - - This method returns the options set or creates the default options for the - descriptor. - """ - if self._options: - return self._options - from google.protobuf import descriptor_pb2 - try: - options_class = getattr(descriptor_pb2, self._options_class_name) - except AttributeError: - raise RuntimeError('Unknown options class name %s!' % - (self._options_class_name)) - self._options = options_class() - return self._options - - -class _NestedDescriptorBase(DescriptorBase): - """Common class for descriptors that can be nested.""" - - def __init__(self, options, options_class_name, name, full_name, - file, containing_type, serialized_start=None, - serialized_end=None): - """Constructor. - - Args: - options: Protocol message options or None - to use default message options. - options_class_name: (str) The class name of the above options. - - name: (str) Name of this protocol message type. - full_name: (str) Fully-qualified name of this protocol message type, - which will include protocol "package" name and the name of any - enclosing types. - file: (FileDescriptor) Reference to file info. - containing_type: if provided, this is a nested descriptor, with this - descriptor as parent, otherwise None. - serialized_start: The start index (inclusive) in block in the - file.serialized_pb that describes this descriptor. - serialized_end: The end index (exclusive) in block in the - file.serialized_pb that describes this descriptor. - """ - super(_NestedDescriptorBase, self).__init__( - options, options_class_name) - - self.name = name - # TODO(falk): Add function to calculate full_name instead of having it in - # memory? - self.full_name = full_name - self.file = file - self.containing_type = containing_type - - self._serialized_start = serialized_start - self._serialized_end = serialized_end - - def GetTopLevelContainingType(self): - """Returns the root if this is a nested type, or itself if its the root.""" - desc = self - while desc.containing_type is not None: - desc = desc.containing_type - return desc - - def CopyToProto(self, proto): - """Copies this to the matching proto in descriptor_pb2. - - Args: - proto: An empty proto instance from descriptor_pb2. - - Raises: - Error: If self couldnt be serialized, due to to few constructor arguments. - """ - if (self.file is not None and - self._serialized_start is not None and - self._serialized_end is not None): - proto.ParseFromString(self.file.serialized_pb[ - self._serialized_start:self._serialized_end]) - else: - raise Error('Descriptor does not contain serialization.') - - -class Descriptor(_NestedDescriptorBase): - - """Descriptor for a protocol message type. - - A Descriptor instance has the following attributes: - - name: (str) Name of this protocol message type. - full_name: (str) Fully-qualified name of this protocol message type, - which will include protocol "package" name and the name of any - enclosing types. - - containing_type: (Descriptor) Reference to the descriptor of the - type containing us, or None if this is top-level. - - fields: (list of FieldDescriptors) Field descriptors for all - fields in this type. - fields_by_number: (dict int -> FieldDescriptor) Same FieldDescriptor - objects as in |fields|, but indexed by "number" attribute in each - FieldDescriptor. - fields_by_name: (dict str -> FieldDescriptor) Same FieldDescriptor - objects as in |fields|, but indexed by "name" attribute in each - FieldDescriptor. - - nested_types: (list of Descriptors) Descriptor references - for all protocol message types nested within this one. - nested_types_by_name: (dict str -> Descriptor) Same Descriptor - objects as in |nested_types|, but indexed by "name" attribute - in each Descriptor. - - enum_types: (list of EnumDescriptors) EnumDescriptor references - for all enums contained within this type. - enum_types_by_name: (dict str ->EnumDescriptor) Same EnumDescriptor - objects as in |enum_types|, but indexed by "name" attribute - in each EnumDescriptor. - enum_values_by_name: (dict str -> EnumValueDescriptor) Dict mapping - from enum value name to EnumValueDescriptor for that value. - - extensions: (list of FieldDescriptor) All extensions defined directly - within this message type (NOT within a nested type). - extensions_by_name: (dict, string -> FieldDescriptor) Same FieldDescriptor - objects as |extensions|, but indexed by "name" attribute of each - FieldDescriptor. - - is_extendable: Does this type define any extension ranges? - - options: (descriptor_pb2.MessageOptions) Protocol message options or None - to use default message options. - - file: (FileDescriptor) Reference to file descriptor. - """ - - def __init__(self, name, full_name, filename, containing_type, fields, - nested_types, enum_types, extensions, options=None, - is_extendable=True, extension_ranges=None, file=None, - serialized_start=None, serialized_end=None): - """Arguments to __init__() are as described in the description - of Descriptor fields above. - - Note that filename is an obsolete argument, that is not used anymore. - Please use file.name to access this as an attribute. - """ - super(Descriptor, self).__init__( - options, 'MessageOptions', name, full_name, file, - containing_type, serialized_start=serialized_start, - serialized_end=serialized_start) - - # We have fields in addition to fields_by_name and fields_by_number, - # so that: - # 1. Clients can index fields by "order in which they're listed." - # 2. Clients can easily iterate over all fields with the terse - # syntax: for f in descriptor.fields: ... - self.fields = fields - for field in self.fields: - field.containing_type = self - self.fields_by_number = dict((f.number, f) for f in fields) - self.fields_by_name = dict((f.name, f) for f in fields) - - self.nested_types = nested_types - self.nested_types_by_name = dict((t.name, t) for t in nested_types) - - self.enum_types = enum_types - for enum_type in self.enum_types: - enum_type.containing_type = self - self.enum_types_by_name = dict((t.name, t) for t in enum_types) - self.enum_values_by_name = dict( - (v.name, v) for t in enum_types for v in t.values) - - self.extensions = extensions - for extension in self.extensions: - extension.extension_scope = self - self.extensions_by_name = dict((f.name, f) for f in extensions) - self.is_extendable = is_extendable - self.extension_ranges = extension_ranges - - self._serialized_start = serialized_start - self._serialized_end = serialized_end - - def EnumValueName(self, enum, value): - """Returns the string name of an enum value. - - This is just a small helper method to simplify a common operation. - - Args: - enum: string name of the Enum. - value: int, value of the enum. - - Returns: - string name of the enum value. - - Raises: - KeyError if either the Enum doesn't exist or the value is not a valid - value for the enum. - """ - return self.enum_types_by_name[enum].values_by_number[value].name - - def CopyToProto(self, proto): - """Copies this to a descriptor_pb2.DescriptorProto. - - Args: - proto: An empty descriptor_pb2.DescriptorProto. - """ - # This function is overriden to give a better doc comment. - super(Descriptor, self).CopyToProto(proto) - - -# TODO(robinson): We should have aggressive checking here, -# for example: -# * If you specify a repeated field, you should not be allowed -# to specify a default value. -# * [Other examples here as needed]. -# -# TODO(robinson): for this and other *Descriptor classes, we -# might also want to lock things down aggressively (e.g., -# prevent clients from setting the attributes). Having -# stronger invariants here in general will reduce the number -# of runtime checks we must do in reflection.py... -class FieldDescriptor(DescriptorBase): - - """Descriptor for a single field in a .proto file. - - A FieldDescriptor instance has the following attributes: - - name: (str) Name of this field, exactly as it appears in .proto. - full_name: (str) Name of this field, including containing scope. This is - particularly relevant for extensions. - index: (int) Dense, 0-indexed index giving the order that this - field textually appears within its message in the .proto file. - number: (int) Tag number declared for this field in the .proto file. - - type: (One of the TYPE_* constants below) Declared type. - cpp_type: (One of the CPPTYPE_* constants below) C++ type used to - represent this field. - - label: (One of the LABEL_* constants below) Tells whether this - field is optional, required, or repeated. - has_default_value: (bool) True if this field has a default value defined, - otherwise false. - default_value: (Varies) Default value of this field. Only - meaningful for non-repeated scalar fields. Repeated fields - should always set this to [], and non-repeated composite - fields should always set this to None. - - containing_type: (Descriptor) Descriptor of the protocol message - type that contains this field. Set by the Descriptor constructor - if we're passed into one. - Somewhat confusingly, for extension fields, this is the - descriptor of the EXTENDED message, not the descriptor - of the message containing this field. (See is_extension and - extension_scope below). - message_type: (Descriptor) If a composite field, a descriptor - of the message type contained in this field. Otherwise, this is None. - enum_type: (EnumDescriptor) If this field contains an enum, a - descriptor of that enum. Otherwise, this is None. - - is_extension: True iff this describes an extension field. - extension_scope: (Descriptor) Only meaningful if is_extension is True. - Gives the message that immediately contains this extension field. - Will be None iff we're a top-level (file-level) extension field. - - options: (descriptor_pb2.FieldOptions) Protocol message field options or - None to use default field options. - """ - - # Must be consistent with C++ FieldDescriptor::Type enum in - # descriptor.h. - # - # TODO(robinson): Find a way to eliminate this repetition. - TYPE_DOUBLE = 1 - TYPE_FLOAT = 2 - TYPE_INT64 = 3 - TYPE_UINT64 = 4 - TYPE_INT32 = 5 - TYPE_FIXED64 = 6 - TYPE_FIXED32 = 7 - TYPE_BOOL = 8 - TYPE_STRING = 9 - TYPE_GROUP = 10 - TYPE_MESSAGE = 11 - TYPE_BYTES = 12 - TYPE_UINT32 = 13 - TYPE_ENUM = 14 - TYPE_SFIXED32 = 15 - TYPE_SFIXED64 = 16 - TYPE_SINT32 = 17 - TYPE_SINT64 = 18 - MAX_TYPE = 18 - - # Must be consistent with C++ FieldDescriptor::CppType enum in - # descriptor.h. - # - # TODO(robinson): Find a way to eliminate this repetition. - CPPTYPE_INT32 = 1 - CPPTYPE_INT64 = 2 - CPPTYPE_UINT32 = 3 - CPPTYPE_UINT64 = 4 - CPPTYPE_DOUBLE = 5 - CPPTYPE_FLOAT = 6 - CPPTYPE_BOOL = 7 - CPPTYPE_ENUM = 8 - CPPTYPE_STRING = 9 - CPPTYPE_MESSAGE = 10 - MAX_CPPTYPE = 10 - - _PYTHON_TO_CPP_PROTO_TYPE_MAP = { - TYPE_DOUBLE: CPPTYPE_DOUBLE, - TYPE_FLOAT: CPPTYPE_FLOAT, - TYPE_ENUM: CPPTYPE_ENUM, - TYPE_INT64: CPPTYPE_INT64, - TYPE_SINT64: CPPTYPE_INT64, - TYPE_SFIXED64: CPPTYPE_INT64, - TYPE_UINT64: CPPTYPE_UINT64, - TYPE_FIXED64: CPPTYPE_UINT64, - TYPE_INT32: CPPTYPE_INT32, - TYPE_SFIXED32: CPPTYPE_INT32, - TYPE_SINT32: CPPTYPE_INT32, - TYPE_UINT32: CPPTYPE_UINT32, - TYPE_FIXED32: CPPTYPE_UINT32, - TYPE_BYTES: CPPTYPE_STRING, - TYPE_STRING: CPPTYPE_STRING, - TYPE_BOOL: CPPTYPE_BOOL, - TYPE_MESSAGE: CPPTYPE_MESSAGE, - TYPE_GROUP: CPPTYPE_MESSAGE - } - - # Must be consistent with C++ FieldDescriptor::Label enum in - # descriptor.h. - # - # TODO(robinson): Find a way to eliminate this repetition. - LABEL_OPTIONAL = 1 - LABEL_REQUIRED = 2 - LABEL_REPEATED = 3 - MAX_LABEL = 3 - - def __init__(self, name, full_name, index, number, type, cpp_type, label, - default_value, message_type, enum_type, containing_type, - is_extension, extension_scope, options=None, - has_default_value=True): - """The arguments are as described in the description of FieldDescriptor - attributes above. - - Note that containing_type may be None, and may be set later if necessary - (to deal with circular references between message types, for example). - Likewise for extension_scope. - """ - super(FieldDescriptor, self).__init__(options, 'FieldOptions') - self.name = name - self.full_name = full_name - self.index = index - self.number = number - self.type = type - self.cpp_type = cpp_type - self.label = label - self.has_default_value = has_default_value - self.default_value = default_value - self.containing_type = containing_type - self.message_type = message_type - self.enum_type = enum_type - self.is_extension = is_extension - self.extension_scope = extension_scope - if api_implementation.Type() == 'cpp': - if is_extension: - if api_implementation.Version() == 2: - self._cdescriptor = _message.GetExtensionDescriptor(full_name) - else: - self._cdescriptor = cpp_message.GetExtensionDescriptor(full_name) - else: - if api_implementation.Version() == 2: - self._cdescriptor = _message.GetFieldDescriptor(full_name) - else: - self._cdescriptor = cpp_message.GetFieldDescriptor(full_name) - else: - self._cdescriptor = None - - @staticmethod - def ProtoTypeToCppProtoType(proto_type): - """Converts from a Python proto type to a C++ Proto Type. - - The Python ProtocolBuffer classes specify both the 'Python' datatype and the - 'C++' datatype - and they're not the same. This helper method should - translate from one to another. - - Args: - proto_type: the Python proto type (descriptor.FieldDescriptor.TYPE_*) - Returns: - descriptor.FieldDescriptor.CPPTYPE_*, the C++ type. - Raises: - TypeTransformationError: when the Python proto type isn't known. - """ - try: - return FieldDescriptor._PYTHON_TO_CPP_PROTO_TYPE_MAP[proto_type] - except KeyError: - raise TypeTransformationError('Unknown proto_type: %s' % proto_type) - - -class EnumDescriptor(_NestedDescriptorBase): - - """Descriptor for an enum defined in a .proto file. - - An EnumDescriptor instance has the following attributes: - - name: (str) Name of the enum type. - full_name: (str) Full name of the type, including package name - and any enclosing type(s). - - values: (list of EnumValueDescriptors) List of the values - in this enum. - values_by_name: (dict str -> EnumValueDescriptor) Same as |values|, - but indexed by the "name" field of each EnumValueDescriptor. - values_by_number: (dict int -> EnumValueDescriptor) Same as |values|, - but indexed by the "number" field of each EnumValueDescriptor. - containing_type: (Descriptor) Descriptor of the immediate containing - type of this enum, or None if this is an enum defined at the - top level in a .proto file. Set by Descriptor's constructor - if we're passed into one. - file: (FileDescriptor) Reference to file descriptor. - options: (descriptor_pb2.EnumOptions) Enum options message or - None to use default enum options. - """ - - def __init__(self, name, full_name, filename, values, - containing_type=None, options=None, file=None, - serialized_start=None, serialized_end=None): - """Arguments are as described in the attribute description above. - - Note that filename is an obsolete argument, that is not used anymore. - Please use file.name to access this as an attribute. - """ - super(EnumDescriptor, self).__init__( - options, 'EnumOptions', name, full_name, file, - containing_type, serialized_start=serialized_start, - serialized_end=serialized_start) - - self.values = values - for value in self.values: - value.type = self - self.values_by_name = dict((v.name, v) for v in values) - self.values_by_number = dict((v.number, v) for v in values) - - self._serialized_start = serialized_start - self._serialized_end = serialized_end - - def CopyToProto(self, proto): - """Copies this to a descriptor_pb2.EnumDescriptorProto. - - Args: - proto: An empty descriptor_pb2.EnumDescriptorProto. - """ - # This function is overriden to give a better doc comment. - super(EnumDescriptor, self).CopyToProto(proto) - - -class EnumValueDescriptor(DescriptorBase): - - """Descriptor for a single value within an enum. - - name: (str) Name of this value. - index: (int) Dense, 0-indexed index giving the order that this - value appears textually within its enum in the .proto file. - number: (int) Actual number assigned to this enum value. - type: (EnumDescriptor) EnumDescriptor to which this value - belongs. Set by EnumDescriptor's constructor if we're - passed into one. - options: (descriptor_pb2.EnumValueOptions) Enum value options message or - None to use default enum value options options. - """ - - def __init__(self, name, index, number, type=None, options=None): - """Arguments are as described in the attribute description above.""" - super(EnumValueDescriptor, self).__init__(options, 'EnumValueOptions') - self.name = name - self.index = index - self.number = number - self.type = type - - -class ServiceDescriptor(_NestedDescriptorBase): - - """Descriptor for a service. - - name: (str) Name of the service. - full_name: (str) Full name of the service, including package name. - index: (int) 0-indexed index giving the order that this services - definition appears withing the .proto file. - methods: (list of MethodDescriptor) List of methods provided by this - service. - options: (descriptor_pb2.ServiceOptions) Service options message or - None to use default service options. - file: (FileDescriptor) Reference to file info. - """ - - def __init__(self, name, full_name, index, methods, options=None, file=None, - serialized_start=None, serialized_end=None): - super(ServiceDescriptor, self).__init__( - options, 'ServiceOptions', name, full_name, file, - None, serialized_start=serialized_start, - serialized_end=serialized_end) - self.index = index - self.methods = methods - # Set the containing service for each method in this service. - for method in self.methods: - method.containing_service = self - - def FindMethodByName(self, name): - """Searches for the specified method, and returns its descriptor.""" - for method in self.methods: - if name == method.name: - return method - return None - - def CopyToProto(self, proto): - """Copies this to a descriptor_pb2.ServiceDescriptorProto. - - Args: - proto: An empty descriptor_pb2.ServiceDescriptorProto. - """ - # This function is overriden to give a better doc comment. - super(ServiceDescriptor, self).CopyToProto(proto) - - -class MethodDescriptor(DescriptorBase): - - """Descriptor for a method in a service. - - name: (str) Name of the method within the service. - full_name: (str) Full name of method. - index: (int) 0-indexed index of the method inside the service. - containing_service: (ServiceDescriptor) The service that contains this - method. - input_type: The descriptor of the message that this method accepts. - output_type: The descriptor of the message that this method returns. - options: (descriptor_pb2.MethodOptions) Method options message or - None to use default method options. - """ - - def __init__(self, name, full_name, index, containing_service, - input_type, output_type, options=None): - """The arguments are as described in the description of MethodDescriptor - attributes above. - - Note that containing_service may be None, and may be set later if necessary. - """ - super(MethodDescriptor, self).__init__(options, 'MethodOptions') - self.name = name - self.full_name = full_name - self.index = index - self.containing_service = containing_service - self.input_type = input_type - self.output_type = output_type - - -class FileDescriptor(DescriptorBase): - """Descriptor for a file. Mimics the descriptor_pb2.FileDescriptorProto. - - name: name of file, relative to root of source tree. - package: name of the package - serialized_pb: (str) Byte string of serialized - descriptor_pb2.FileDescriptorProto. - """ - - def __init__(self, name, package, options=None, serialized_pb=None): - """Constructor.""" - super(FileDescriptor, self).__init__(options, 'FileOptions') - - self.message_types_by_name = {} - self.name = name - self.package = package - self.serialized_pb = serialized_pb - if (api_implementation.Type() == 'cpp' and - self.serialized_pb is not None): - if api_implementation.Version() == 2: - _message.BuildFile(self.serialized_pb) - else: - cpp_message.BuildFile(self.serialized_pb) - - def CopyToProto(self, proto): - """Copies this to a descriptor_pb2.FileDescriptorProto. - - Args: - proto: An empty descriptor_pb2.FileDescriptorProto. - """ - proto.ParseFromString(self.serialized_pb) - - -def _ParseOptions(message, string): - """Parses serialized options. - - This helper function is used to parse serialized options in generated - proto2 files. It must not be used outside proto2. - """ - message.ParseFromString(string) - return message - - -def MakeDescriptor(desc_proto, package=''): - """Make a protobuf Descriptor given a DescriptorProto protobuf. - - Args: - desc_proto: The descriptor_pb2.DescriptorProto protobuf message. - package: Optional package name for the new message Descriptor (string). - - Returns: - A Descriptor for protobuf messages. - """ - full_message_name = [desc_proto.name] - if package: full_message_name.insert(0, package) - fields = [] - for field_proto in desc_proto.field: - full_name = '.'.join(full_message_name + [field_proto.name]) - field = FieldDescriptor( - field_proto.name, full_name, field_proto.number - 1, - field_proto.number, field_proto.type, - FieldDescriptor.ProtoTypeToCppProtoType(field_proto.type), - field_proto.label, None, None, None, None, False, None, - has_default_value=False) - fields.append(field) - - desc_name = '.'.join(full_message_name) - return Descriptor(desc_proto.name, desc_name, None, None, fields, - [], [], []) diff --git a/generator/google/protobuf/descriptor_database.py b/generator/google/protobuf/descriptor_database.py deleted file mode 100644 index 8665d3c..0000000 --- a/generator/google/protobuf/descriptor_database.py +++ /dev/null @@ -1,120 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Provides a container for DescriptorProtos.""" - -__author__ = 'matthewtoia@google.com (Matt Toia)' - - -class DescriptorDatabase(object): - """A container accepting FileDescriptorProtos and maps DescriptorProtos.""" - - def __init__(self): - self._file_desc_protos_by_file = {} - self._file_desc_protos_by_symbol = {} - - def Add(self, file_desc_proto): - """Adds the FileDescriptorProto and its types to this database. - - Args: - file_desc_proto: The FileDescriptorProto to add. - """ - - self._file_desc_protos_by_file[file_desc_proto.name] = file_desc_proto - package = file_desc_proto.package - for message in file_desc_proto.message_type: - self._file_desc_protos_by_symbol.update( - (name, file_desc_proto) for name in _ExtractSymbols(message, package)) - for enum in file_desc_proto.enum_type: - self._file_desc_protos_by_symbol[ - '.'.join((package, enum.name))] = file_desc_proto - - def FindFileByName(self, name): - """Finds the file descriptor proto by file name. - - Typically the file name is a relative path ending to a .proto file. The - proto with the given name will have to have been added to this database - using the Add method or else an error will be raised. - - Args: - name: The file name to find. - - Returns: - The file descriptor proto matching the name. - - Raises: - KeyError if no file by the given name was added. - """ - - return self._file_desc_protos_by_file[name] - - def FindFileContainingSymbol(self, symbol): - """Finds the file descriptor proto containing the specified symbol. - - The symbol should be a fully qualified name including the file descriptor's - package and any containing messages. Some examples: - - 'some.package.name.Message' - 'some.package.name.Message.NestedEnum' - - The file descriptor proto containing the specified symbol must be added to - this database using the Add method or else an error will be raised. - - Args: - symbol: The fully qualified symbol name. - - Returns: - The file descriptor proto containing the symbol. - - Raises: - KeyError if no file contains the specified symbol. - """ - - return self._file_desc_protos_by_symbol[symbol] - - -def _ExtractSymbols(desc_proto, package): - """Pulls out all the symbols from a descriptor proto. - - Args: - desc_proto: The proto to extract symbols from. - package: The package containing the descriptor type. - - Yields: - The fully qualified name found in the descriptor. - """ - - message_name = '.'.join((package, desc_proto.name)) - yield message_name - for nested_type in desc_proto.nested_type: - for symbol in _ExtractSymbols(nested_type, message_name): - yield symbol - for enum_type in desc_proto.enum_type: - yield '.'.join((message_name, enum_type.name)) diff --git a/generator/google/protobuf/descriptor_pb2.py b/generator/google/protobuf/descriptor_pb2.py deleted file mode 100644 index 20b2d94..0000000 --- a/generator/google/protobuf/descriptor_pb2.py +++ /dev/null @@ -1,1346 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: google/protobuf/descriptor.proto - -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -# @@protoc_insertion_point(imports) - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='google/protobuf/descriptor.proto', - package='google.protobuf', - serialized_pb='\n google/protobuf/descriptor.proto\x12\x0fgoogle.protobuf\"G\n\x11\x46ileDescriptorSet\x12\x32\n\x04\x66ile\x18\x01 \x03(\x0b\x32$.google.protobuf.FileDescriptorProto\"\xcb\x03\n\x13\x46ileDescriptorProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07package\x18\x02 \x01(\t\x12\x12\n\ndependency\x18\x03 \x03(\t\x12\x19\n\x11public_dependency\x18\n \x03(\x05\x12\x17\n\x0fweak_dependency\x18\x0b \x03(\x05\x12\x36\n\x0cmessage_type\x18\x04 \x03(\x0b\x32 .google.protobuf.DescriptorProto\x12\x37\n\tenum_type\x18\x05 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProto\x12\x38\n\x07service\x18\x06 \x03(\x0b\x32\'.google.protobuf.ServiceDescriptorProto\x12\x38\n\textension\x18\x07 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProto\x12-\n\x07options\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.FileOptions\x12\x39\n\x10source_code_info\x18\t \x01(\x0b\x32\x1f.google.protobuf.SourceCodeInfo\"\xa9\x03\n\x0f\x44\x65scriptorProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x34\n\x05\x66ield\x18\x02 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProto\x12\x38\n\textension\x18\x06 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProto\x12\x35\n\x0bnested_type\x18\x03 \x03(\x0b\x32 .google.protobuf.DescriptorProto\x12\x37\n\tenum_type\x18\x04 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProto\x12H\n\x0f\x65xtension_range\x18\x05 \x03(\x0b\x32/.google.protobuf.DescriptorProto.ExtensionRange\x12\x30\n\x07options\x18\x07 \x01(\x0b\x32\x1f.google.protobuf.MessageOptions\x1a,\n\x0e\x45xtensionRange\x12\r\n\x05start\x18\x01 \x01(\x05\x12\x0b\n\x03\x65nd\x18\x02 \x01(\x05\"\x94\x05\n\x14\x46ieldDescriptorProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06number\x18\x03 \x01(\x05\x12:\n\x05label\x18\x04 \x01(\x0e\x32+.google.protobuf.FieldDescriptorProto.Label\x12\x38\n\x04type\x18\x05 \x01(\x0e\x32*.google.protobuf.FieldDescriptorProto.Type\x12\x11\n\ttype_name\x18\x06 \x01(\t\x12\x10\n\x08\x65xtendee\x18\x02 \x01(\t\x12\x15\n\rdefault_value\x18\x07 \x01(\t\x12.\n\x07options\x18\x08 \x01(\x0b\x32\x1d.google.protobuf.FieldOptions\"\xb6\x02\n\x04Type\x12\x0f\n\x0bTYPE_DOUBLE\x10\x01\x12\x0e\n\nTYPE_FLOAT\x10\x02\x12\x0e\n\nTYPE_INT64\x10\x03\x12\x0f\n\x0bTYPE_UINT64\x10\x04\x12\x0e\n\nTYPE_INT32\x10\x05\x12\x10\n\x0cTYPE_FIXED64\x10\x06\x12\x10\n\x0cTYPE_FIXED32\x10\x07\x12\r\n\tTYPE_BOOL\x10\x08\x12\x0f\n\x0bTYPE_STRING\x10\t\x12\x0e\n\nTYPE_GROUP\x10\n\x12\x10\n\x0cTYPE_MESSAGE\x10\x0b\x12\x0e\n\nTYPE_BYTES\x10\x0c\x12\x0f\n\x0bTYPE_UINT32\x10\r\x12\r\n\tTYPE_ENUM\x10\x0e\x12\x11\n\rTYPE_SFIXED32\x10\x0f\x12\x11\n\rTYPE_SFIXED64\x10\x10\x12\x0f\n\x0bTYPE_SINT32\x10\x11\x12\x0f\n\x0bTYPE_SINT64\x10\x12\"C\n\x05Label\x12\x12\n\x0eLABEL_OPTIONAL\x10\x01\x12\x12\n\x0eLABEL_REQUIRED\x10\x02\x12\x12\n\x0eLABEL_REPEATED\x10\x03\"\x8c\x01\n\x13\x45numDescriptorProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x38\n\x05value\x18\x02 \x03(\x0b\x32).google.protobuf.EnumValueDescriptorProto\x12-\n\x07options\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.EnumOptions\"l\n\x18\x45numValueDescriptorProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06number\x18\x02 \x01(\x05\x12\x32\n\x07options\x18\x03 \x01(\x0b\x32!.google.protobuf.EnumValueOptions\"\x90\x01\n\x16ServiceDescriptorProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x36\n\x06method\x18\x02 \x03(\x0b\x32&.google.protobuf.MethodDescriptorProto\x12\x30\n\x07options\x18\x03 \x01(\x0b\x32\x1f.google.protobuf.ServiceOptions\"\x7f\n\x15MethodDescriptorProto\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x12\n\ninput_type\x18\x02 \x01(\t\x12\x13\n\x0boutput_type\x18\x03 \x01(\t\x12/\n\x07options\x18\x04 \x01(\x0b\x32\x1e.google.protobuf.MethodOptions\"\xe9\x03\n\x0b\x46ileOptions\x12\x14\n\x0cjava_package\x18\x01 \x01(\t\x12\x1c\n\x14java_outer_classname\x18\x08 \x01(\t\x12\"\n\x13java_multiple_files\x18\n \x01(\x08:\x05\x66\x61lse\x12,\n\x1djava_generate_equals_and_hash\x18\x14 \x01(\x08:\x05\x66\x61lse\x12\x46\n\x0coptimize_for\x18\t \x01(\x0e\x32).google.protobuf.FileOptions.OptimizeMode:\x05SPEED\x12\x12\n\ngo_package\x18\x0b \x01(\t\x12\"\n\x13\x63\x63_generic_services\x18\x10 \x01(\x08:\x05\x66\x61lse\x12$\n\x15java_generic_services\x18\x11 \x01(\x08:\x05\x66\x61lse\x12\"\n\x13py_generic_services\x18\x12 \x01(\x08:\x05\x66\x61lse\x12\x43\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOption\":\n\x0cOptimizeMode\x12\t\n\x05SPEED\x10\x01\x12\r\n\tCODE_SIZE\x10\x02\x12\x10\n\x0cLITE_RUNTIME\x10\x03*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xb8\x01\n\x0eMessageOptions\x12&\n\x17message_set_wire_format\x18\x01 \x01(\x08:\x05\x66\x61lse\x12.\n\x1fno_standard_descriptor_accessor\x18\x02 \x01(\x08:\x05\x66\x61lse\x12\x43\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xbe\x02\n\x0c\x46ieldOptions\x12:\n\x05\x63type\x18\x01 \x01(\x0e\x32#.google.protobuf.FieldOptions.CType:\x06STRING\x12\x0e\n\x06packed\x18\x02 \x01(\x08\x12\x13\n\x04lazy\x18\x05 \x01(\x08:\x05\x66\x61lse\x12\x19\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lse\x12\x1c\n\x14\x65xperimental_map_key\x18\t \x01(\t\x12\x13\n\x04weak\x18\n \x01(\x08:\x05\x66\x61lse\x12\x43\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOption\"/\n\x05\x43Type\x12\n\n\x06STRING\x10\x00\x12\x08\n\x04\x43ORD\x10\x01\x12\x10\n\x0cSTRING_PIECE\x10\x02*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"x\n\x0b\x45numOptions\x12\x19\n\x0b\x61llow_alias\x18\x02 \x01(\x08:\x04true\x12\x43\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"b\n\x10\x45numValueOptions\x12\x43\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"`\n\x0eServiceOptions\x12\x43\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"_\n\rMethodOptions\x12\x43\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x9e\x02\n\x13UninterpretedOption\x12;\n\x04name\x18\x02 \x03(\x0b\x32-.google.protobuf.UninterpretedOption.NamePart\x12\x18\n\x10identifier_value\x18\x03 \x01(\t\x12\x1a\n\x12positive_int_value\x18\x04 \x01(\x04\x12\x1a\n\x12negative_int_value\x18\x05 \x01(\x03\x12\x14\n\x0c\x64ouble_value\x18\x06 \x01(\x01\x12\x14\n\x0cstring_value\x18\x07 \x01(\x0c\x12\x17\n\x0f\x61ggregate_value\x18\x08 \x01(\t\x1a\x33\n\x08NamePart\x12\x11\n\tname_part\x18\x01 \x02(\t\x12\x14\n\x0cis_extension\x18\x02 \x02(\x08\"\xb1\x01\n\x0eSourceCodeInfo\x12:\n\x08location\x18\x01 \x03(\x0b\x32(.google.protobuf.SourceCodeInfo.Location\x1a\x63\n\x08Location\x12\x10\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01\x12\x10\n\x04span\x18\x02 \x03(\x05\x42\x02\x10\x01\x12\x18\n\x10leading_comments\x18\x03 \x01(\t\x12\x19\n\x11trailing_comments\x18\x04 \x01(\tB)\n\x13\x63om.google.protobufB\x10\x44\x65scriptorProtosH\x01') - - - -_FIELDDESCRIPTORPROTO_TYPE = _descriptor.EnumDescriptor( - name='Type', - full_name='google.protobuf.FieldDescriptorProto.Type', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='TYPE_DOUBLE', index=0, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_FLOAT', index=1, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_INT64', index=2, number=3, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_UINT64', index=3, number=4, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_INT32', index=4, number=5, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_FIXED64', index=5, number=6, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_FIXED32', index=6, number=7, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_BOOL', index=7, number=8, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_STRING', index=8, number=9, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_GROUP', index=9, number=10, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_MESSAGE', index=10, number=11, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_BYTES', index=11, number=12, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_UINT32', index=12, number=13, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_ENUM', index=13, number=14, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_SFIXED32', index=14, number=15, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_SFIXED64', index=15, number=16, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_SINT32', index=16, number=17, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='TYPE_SINT64', index=17, number=18, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=1298, - serialized_end=1608, -) - -_FIELDDESCRIPTORPROTO_LABEL = _descriptor.EnumDescriptor( - name='Label', - full_name='google.protobuf.FieldDescriptorProto.Label', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='LABEL_OPTIONAL', index=0, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LABEL_REQUIRED', index=1, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LABEL_REPEATED', index=2, number=3, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=1610, - serialized_end=1677, -) - -_FILEOPTIONS_OPTIMIZEMODE = _descriptor.EnumDescriptor( - name='OptimizeMode', - full_name='google.protobuf.FileOptions.OptimizeMode', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='SPEED', index=0, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CODE_SIZE', index=1, number=2, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='LITE_RUNTIME', index=2, number=3, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=2629, - serialized_end=2687, -) - -_FIELDOPTIONS_CTYPE = _descriptor.EnumDescriptor( - name='CType', - full_name='google.protobuf.FieldOptions.CType', - filename=None, - file=DESCRIPTOR, - values=[ - _descriptor.EnumValueDescriptor( - name='STRING', index=0, number=0, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='CORD', index=1, number=1, - options=None, - type=None), - _descriptor.EnumValueDescriptor( - name='STRING_PIECE', index=2, number=2, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=3148, - serialized_end=3195, -) - - -_FILEDESCRIPTORSET = _descriptor.Descriptor( - name='FileDescriptorSet', - full_name='google.protobuf.FileDescriptorSet', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='file', full_name='google.protobuf.FileDescriptorSet.file', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=53, - serialized_end=124, -) - - -_FILEDESCRIPTORPROTO = _descriptor.Descriptor( - name='FileDescriptorProto', - full_name='google.protobuf.FileDescriptorProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.protobuf.FileDescriptorProto.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='package', full_name='google.protobuf.FileDescriptorProto.package', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='dependency', full_name='google.protobuf.FileDescriptorProto.dependency', index=2, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='public_dependency', full_name='google.protobuf.FileDescriptorProto.public_dependency', index=3, - number=10, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='weak_dependency', full_name='google.protobuf.FileDescriptorProto.weak_dependency', index=4, - number=11, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='message_type', full_name='google.protobuf.FileDescriptorProto.message_type', index=5, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='enum_type', full_name='google.protobuf.FileDescriptorProto.enum_type', index=6, - number=5, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='service', full_name='google.protobuf.FileDescriptorProto.service', index=7, - number=6, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='extension', full_name='google.protobuf.FileDescriptorProto.extension', index=8, - number=7, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='options', full_name='google.protobuf.FileDescriptorProto.options', index=9, - number=8, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='source_code_info', full_name='google.protobuf.FileDescriptorProto.source_code_info', index=10, - number=9, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=127, - serialized_end=586, -) - - -_DESCRIPTORPROTO_EXTENSIONRANGE = _descriptor.Descriptor( - name='ExtensionRange', - full_name='google.protobuf.DescriptorProto.ExtensionRange', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='start', full_name='google.protobuf.DescriptorProto.ExtensionRange.start', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='end', full_name='google.protobuf.DescriptorProto.ExtensionRange.end', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=970, - serialized_end=1014, -) - -_DESCRIPTORPROTO = _descriptor.Descriptor( - name='DescriptorProto', - full_name='google.protobuf.DescriptorProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.protobuf.DescriptorProto.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='field', full_name='google.protobuf.DescriptorProto.field', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='extension', full_name='google.protobuf.DescriptorProto.extension', index=2, - number=6, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='nested_type', full_name='google.protobuf.DescriptorProto.nested_type', index=3, - number=3, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='enum_type', full_name='google.protobuf.DescriptorProto.enum_type', index=4, - number=4, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='extension_range', full_name='google.protobuf.DescriptorProto.extension_range', index=5, - number=5, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='options', full_name='google.protobuf.DescriptorProto.options', index=6, - number=7, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_DESCRIPTORPROTO_EXTENSIONRANGE, ], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=589, - serialized_end=1014, -) - - -_FIELDDESCRIPTORPROTO = _descriptor.Descriptor( - name='FieldDescriptorProto', - full_name='google.protobuf.FieldDescriptorProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.protobuf.FieldDescriptorProto.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='number', full_name='google.protobuf.FieldDescriptorProto.number', index=1, - number=3, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='label', full_name='google.protobuf.FieldDescriptorProto.label', index=2, - number=4, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='type', full_name='google.protobuf.FieldDescriptorProto.type', index=3, - number=5, type=14, cpp_type=8, label=1, - has_default_value=False, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='type_name', full_name='google.protobuf.FieldDescriptorProto.type_name', index=4, - number=6, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='extendee', full_name='google.protobuf.FieldDescriptorProto.extendee', index=5, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='default_value', full_name='google.protobuf.FieldDescriptorProto.default_value', index=6, - number=7, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='options', full_name='google.protobuf.FieldDescriptorProto.options', index=7, - number=8, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _FIELDDESCRIPTORPROTO_TYPE, - _FIELDDESCRIPTORPROTO_LABEL, - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1017, - serialized_end=1677, -) - - -_ENUMDESCRIPTORPROTO = _descriptor.Descriptor( - name='EnumDescriptorProto', - full_name='google.protobuf.EnumDescriptorProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.protobuf.EnumDescriptorProto.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='value', full_name='google.protobuf.EnumDescriptorProto.value', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='options', full_name='google.protobuf.EnumDescriptorProto.options', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1680, - serialized_end=1820, -) - - -_ENUMVALUEDESCRIPTORPROTO = _descriptor.Descriptor( - name='EnumValueDescriptorProto', - full_name='google.protobuf.EnumValueDescriptorProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.protobuf.EnumValueDescriptorProto.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='number', full_name='google.protobuf.EnumValueDescriptorProto.number', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='options', full_name='google.protobuf.EnumValueDescriptorProto.options', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1822, - serialized_end=1930, -) - - -_SERVICEDESCRIPTORPROTO = _descriptor.Descriptor( - name='ServiceDescriptorProto', - full_name='google.protobuf.ServiceDescriptorProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.protobuf.ServiceDescriptorProto.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='method', full_name='google.protobuf.ServiceDescriptorProto.method', index=1, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='options', full_name='google.protobuf.ServiceDescriptorProto.options', index=2, - number=3, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=1933, - serialized_end=2077, -) - - -_METHODDESCRIPTORPROTO = _descriptor.Descriptor( - name='MethodDescriptorProto', - full_name='google.protobuf.MethodDescriptorProto', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.protobuf.MethodDescriptorProto.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='input_type', full_name='google.protobuf.MethodDescriptorProto.input_type', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='output_type', full_name='google.protobuf.MethodDescriptorProto.output_type', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='options', full_name='google.protobuf.MethodDescriptorProto.options', index=3, - number=4, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=2079, - serialized_end=2206, -) - - -_FILEOPTIONS = _descriptor.Descriptor( - name='FileOptions', - full_name='google.protobuf.FileOptions', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='java_package', full_name='google.protobuf.FileOptions.java_package', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='java_outer_classname', full_name='google.protobuf.FileOptions.java_outer_classname', index=1, - number=8, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='java_multiple_files', full_name='google.protobuf.FileOptions.java_multiple_files', index=2, - number=10, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='java_generate_equals_and_hash', full_name='google.protobuf.FileOptions.java_generate_equals_and_hash', index=3, - number=20, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='optimize_for', full_name='google.protobuf.FileOptions.optimize_for', index=4, - number=9, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=1, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='go_package', full_name='google.protobuf.FileOptions.go_package', index=5, - number=11, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='cc_generic_services', full_name='google.protobuf.FileOptions.cc_generic_services', index=6, - number=16, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='java_generic_services', full_name='google.protobuf.FileOptions.java_generic_services', index=7, - number=17, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='py_generic_services', full_name='google.protobuf.FileOptions.py_generic_services', index=8, - number=18, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='uninterpreted_option', full_name='google.protobuf.FileOptions.uninterpreted_option', index=9, - number=999, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _FILEOPTIONS_OPTIMIZEMODE, - ], - options=None, - is_extendable=True, - extension_ranges=[(1000, 536870912), ], - serialized_start=2209, - serialized_end=2698, -) - - -_MESSAGEOPTIONS = _descriptor.Descriptor( - name='MessageOptions', - full_name='google.protobuf.MessageOptions', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='message_set_wire_format', full_name='google.protobuf.MessageOptions.message_set_wire_format', index=0, - number=1, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='no_standard_descriptor_accessor', full_name='google.protobuf.MessageOptions.no_standard_descriptor_accessor', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='uninterpreted_option', full_name='google.protobuf.MessageOptions.uninterpreted_option', index=2, - number=999, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=True, - extension_ranges=[(1000, 536870912), ], - serialized_start=2701, - serialized_end=2885, -) - - -_FIELDOPTIONS = _descriptor.Descriptor( - name='FieldOptions', - full_name='google.protobuf.FieldOptions', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='ctype', full_name='google.protobuf.FieldOptions.ctype', index=0, - number=1, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='packed', full_name='google.protobuf.FieldOptions.packed', index=1, - number=2, type=8, cpp_type=7, label=1, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='lazy', full_name='google.protobuf.FieldOptions.lazy', index=2, - number=5, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='deprecated', full_name='google.protobuf.FieldOptions.deprecated', index=3, - number=3, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='experimental_map_key', full_name='google.protobuf.FieldOptions.experimental_map_key', index=4, - number=9, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='weak', full_name='google.protobuf.FieldOptions.weak', index=5, - number=10, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='uninterpreted_option', full_name='google.protobuf.FieldOptions.uninterpreted_option', index=6, - number=999, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - _FIELDOPTIONS_CTYPE, - ], - options=None, - is_extendable=True, - extension_ranges=[(1000, 536870912), ], - serialized_start=2888, - serialized_end=3206, -) - - -_ENUMOPTIONS = _descriptor.Descriptor( - name='EnumOptions', - full_name='google.protobuf.EnumOptions', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='allow_alias', full_name='google.protobuf.EnumOptions.allow_alias', index=0, - number=2, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='uninterpreted_option', full_name='google.protobuf.EnumOptions.uninterpreted_option', index=1, - number=999, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=True, - extension_ranges=[(1000, 536870912), ], - serialized_start=3208, - serialized_end=3328, -) - - -_ENUMVALUEOPTIONS = _descriptor.Descriptor( - name='EnumValueOptions', - full_name='google.protobuf.EnumValueOptions', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uninterpreted_option', full_name='google.protobuf.EnumValueOptions.uninterpreted_option', index=0, - number=999, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=True, - extension_ranges=[(1000, 536870912), ], - serialized_start=3330, - serialized_end=3428, -) - - -_SERVICEOPTIONS = _descriptor.Descriptor( - name='ServiceOptions', - full_name='google.protobuf.ServiceOptions', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uninterpreted_option', full_name='google.protobuf.ServiceOptions.uninterpreted_option', index=0, - number=999, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=True, - extension_ranges=[(1000, 536870912), ], - serialized_start=3430, - serialized_end=3526, -) - - -_METHODOPTIONS = _descriptor.Descriptor( - name='MethodOptions', - full_name='google.protobuf.MethodOptions', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='uninterpreted_option', full_name='google.protobuf.MethodOptions.uninterpreted_option', index=0, - number=999, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=True, - extension_ranges=[(1000, 536870912), ], - serialized_start=3528, - serialized_end=3623, -) - - -_UNINTERPRETEDOPTION_NAMEPART = _descriptor.Descriptor( - name='NamePart', - full_name='google.protobuf.UninterpretedOption.NamePart', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name_part', full_name='google.protobuf.UninterpretedOption.NamePart.name_part', index=0, - number=1, type=9, cpp_type=9, label=2, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='is_extension', full_name='google.protobuf.UninterpretedOption.NamePart.is_extension', index=1, - number=2, type=8, cpp_type=7, label=2, - has_default_value=False, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=3861, - serialized_end=3912, -) - -_UNINTERPRETEDOPTION = _descriptor.Descriptor( - name='UninterpretedOption', - full_name='google.protobuf.UninterpretedOption', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='name', full_name='google.protobuf.UninterpretedOption.name', index=0, - number=2, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='identifier_value', full_name='google.protobuf.UninterpretedOption.identifier_value', index=1, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='positive_int_value', full_name='google.protobuf.UninterpretedOption.positive_int_value', index=2, - number=4, type=4, cpp_type=4, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='negative_int_value', full_name='google.protobuf.UninterpretedOption.negative_int_value', index=3, - number=5, type=3, cpp_type=2, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='double_value', full_name='google.protobuf.UninterpretedOption.double_value', index=4, - number=6, type=1, cpp_type=5, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='string_value', full_name='google.protobuf.UninterpretedOption.string_value', index=5, - number=7, type=12, cpp_type=9, label=1, - has_default_value=False, default_value="", - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='aggregate_value', full_name='google.protobuf.UninterpretedOption.aggregate_value', index=6, - number=8, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_UNINTERPRETEDOPTION_NAMEPART, ], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=3626, - serialized_end=3912, -) - - -_SOURCECODEINFO_LOCATION = _descriptor.Descriptor( - name='Location', - full_name='google.protobuf.SourceCodeInfo.Location', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='path', full_name='google.protobuf.SourceCodeInfo.Location.path', index=0, - number=1, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='span', full_name='google.protobuf.SourceCodeInfo.Location.span', index=1, - number=2, type=5, cpp_type=1, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='leading_comments', full_name='google.protobuf.SourceCodeInfo.Location.leading_comments', index=2, - number=3, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='trailing_comments', full_name='google.protobuf.SourceCodeInfo.Location.trailing_comments', index=3, - number=4, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=3993, - serialized_end=4092, -) - -_SOURCECODEINFO = _descriptor.Descriptor( - name='SourceCodeInfo', - full_name='google.protobuf.SourceCodeInfo', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='location', full_name='google.protobuf.SourceCodeInfo.location', index=0, - number=1, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_SOURCECODEINFO_LOCATION, ], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=3915, - serialized_end=4092, -) - -_FILEDESCRIPTORSET.fields_by_name['file'].message_type = _FILEDESCRIPTORPROTO -_FILEDESCRIPTORPROTO.fields_by_name['message_type'].message_type = _DESCRIPTORPROTO -_FILEDESCRIPTORPROTO.fields_by_name['enum_type'].message_type = _ENUMDESCRIPTORPROTO -_FILEDESCRIPTORPROTO.fields_by_name['service'].message_type = _SERVICEDESCRIPTORPROTO -_FILEDESCRIPTORPROTO.fields_by_name['extension'].message_type = _FIELDDESCRIPTORPROTO -_FILEDESCRIPTORPROTO.fields_by_name['options'].message_type = _FILEOPTIONS -_FILEDESCRIPTORPROTO.fields_by_name['source_code_info'].message_type = _SOURCECODEINFO -_DESCRIPTORPROTO_EXTENSIONRANGE.containing_type = _DESCRIPTORPROTO; -_DESCRIPTORPROTO.fields_by_name['field'].message_type = _FIELDDESCRIPTORPROTO -_DESCRIPTORPROTO.fields_by_name['extension'].message_type = _FIELDDESCRIPTORPROTO -_DESCRIPTORPROTO.fields_by_name['nested_type'].message_type = _DESCRIPTORPROTO -_DESCRIPTORPROTO.fields_by_name['enum_type'].message_type = _ENUMDESCRIPTORPROTO -_DESCRIPTORPROTO.fields_by_name['extension_range'].message_type = _DESCRIPTORPROTO_EXTENSIONRANGE -_DESCRIPTORPROTO.fields_by_name['options'].message_type = _MESSAGEOPTIONS -_FIELDDESCRIPTORPROTO.fields_by_name['label'].enum_type = _FIELDDESCRIPTORPROTO_LABEL -_FIELDDESCRIPTORPROTO.fields_by_name['type'].enum_type = _FIELDDESCRIPTORPROTO_TYPE -_FIELDDESCRIPTORPROTO.fields_by_name['options'].message_type = _FIELDOPTIONS -_FIELDDESCRIPTORPROTO_TYPE.containing_type = _FIELDDESCRIPTORPROTO; -_FIELDDESCRIPTORPROTO_LABEL.containing_type = _FIELDDESCRIPTORPROTO; -_ENUMDESCRIPTORPROTO.fields_by_name['value'].message_type = _ENUMVALUEDESCRIPTORPROTO -_ENUMDESCRIPTORPROTO.fields_by_name['options'].message_type = _ENUMOPTIONS -_ENUMVALUEDESCRIPTORPROTO.fields_by_name['options'].message_type = _ENUMVALUEOPTIONS -_SERVICEDESCRIPTORPROTO.fields_by_name['method'].message_type = _METHODDESCRIPTORPROTO -_SERVICEDESCRIPTORPROTO.fields_by_name['options'].message_type = _SERVICEOPTIONS -_METHODDESCRIPTORPROTO.fields_by_name['options'].message_type = _METHODOPTIONS -_FILEOPTIONS.fields_by_name['optimize_for'].enum_type = _FILEOPTIONS_OPTIMIZEMODE -_FILEOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION -_FILEOPTIONS_OPTIMIZEMODE.containing_type = _FILEOPTIONS; -_MESSAGEOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION -_FIELDOPTIONS.fields_by_name['ctype'].enum_type = _FIELDOPTIONS_CTYPE -_FIELDOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION -_FIELDOPTIONS_CTYPE.containing_type = _FIELDOPTIONS; -_ENUMOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION -_ENUMVALUEOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION -_SERVICEOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION -_METHODOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION -_UNINTERPRETEDOPTION_NAMEPART.containing_type = _UNINTERPRETEDOPTION; -_UNINTERPRETEDOPTION.fields_by_name['name'].message_type = _UNINTERPRETEDOPTION_NAMEPART -_SOURCECODEINFO_LOCATION.containing_type = _SOURCECODEINFO; -_SOURCECODEINFO.fields_by_name['location'].message_type = _SOURCECODEINFO_LOCATION -DESCRIPTOR.message_types_by_name['FileDescriptorSet'] = _FILEDESCRIPTORSET -DESCRIPTOR.message_types_by_name['FileDescriptorProto'] = _FILEDESCRIPTORPROTO -DESCRIPTOR.message_types_by_name['DescriptorProto'] = _DESCRIPTORPROTO -DESCRIPTOR.message_types_by_name['FieldDescriptorProto'] = _FIELDDESCRIPTORPROTO -DESCRIPTOR.message_types_by_name['EnumDescriptorProto'] = _ENUMDESCRIPTORPROTO -DESCRIPTOR.message_types_by_name['EnumValueDescriptorProto'] = _ENUMVALUEDESCRIPTORPROTO -DESCRIPTOR.message_types_by_name['ServiceDescriptorProto'] = _SERVICEDESCRIPTORPROTO -DESCRIPTOR.message_types_by_name['MethodDescriptorProto'] = _METHODDESCRIPTORPROTO -DESCRIPTOR.message_types_by_name['FileOptions'] = _FILEOPTIONS -DESCRIPTOR.message_types_by_name['MessageOptions'] = _MESSAGEOPTIONS -DESCRIPTOR.message_types_by_name['FieldOptions'] = _FIELDOPTIONS -DESCRIPTOR.message_types_by_name['EnumOptions'] = _ENUMOPTIONS -DESCRIPTOR.message_types_by_name['EnumValueOptions'] = _ENUMVALUEOPTIONS -DESCRIPTOR.message_types_by_name['ServiceOptions'] = _SERVICEOPTIONS -DESCRIPTOR.message_types_by_name['MethodOptions'] = _METHODOPTIONS -DESCRIPTOR.message_types_by_name['UninterpretedOption'] = _UNINTERPRETEDOPTION -DESCRIPTOR.message_types_by_name['SourceCodeInfo'] = _SOURCECODEINFO - -class FileDescriptorSet(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _FILEDESCRIPTORSET - - # @@protoc_insertion_point(class_scope:google.protobuf.FileDescriptorSet) - -class FileDescriptorProto(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _FILEDESCRIPTORPROTO - - # @@protoc_insertion_point(class_scope:google.protobuf.FileDescriptorProto) - -class DescriptorProto(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - - class ExtensionRange(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _DESCRIPTORPROTO_EXTENSIONRANGE - - # @@protoc_insertion_point(class_scope:google.protobuf.DescriptorProto.ExtensionRange) - DESCRIPTOR = _DESCRIPTORPROTO - - # @@protoc_insertion_point(class_scope:google.protobuf.DescriptorProto) - -class FieldDescriptorProto(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _FIELDDESCRIPTORPROTO - - # @@protoc_insertion_point(class_scope:google.protobuf.FieldDescriptorProto) - -class EnumDescriptorProto(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _ENUMDESCRIPTORPROTO - - # @@protoc_insertion_point(class_scope:google.protobuf.EnumDescriptorProto) - -class EnumValueDescriptorProto(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _ENUMVALUEDESCRIPTORPROTO - - # @@protoc_insertion_point(class_scope:google.protobuf.EnumValueDescriptorProto) - -class ServiceDescriptorProto(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _SERVICEDESCRIPTORPROTO - - # @@protoc_insertion_point(class_scope:google.protobuf.ServiceDescriptorProto) - -class MethodDescriptorProto(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _METHODDESCRIPTORPROTO - - # @@protoc_insertion_point(class_scope:google.protobuf.MethodDescriptorProto) - -class FileOptions(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _FILEOPTIONS - - # @@protoc_insertion_point(class_scope:google.protobuf.FileOptions) - -class MessageOptions(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _MESSAGEOPTIONS - - # @@protoc_insertion_point(class_scope:google.protobuf.MessageOptions) - -class FieldOptions(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _FIELDOPTIONS - - # @@protoc_insertion_point(class_scope:google.protobuf.FieldOptions) - -class EnumOptions(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _ENUMOPTIONS - - # @@protoc_insertion_point(class_scope:google.protobuf.EnumOptions) - -class EnumValueOptions(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _ENUMVALUEOPTIONS - - # @@protoc_insertion_point(class_scope:google.protobuf.EnumValueOptions) - -class ServiceOptions(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _SERVICEOPTIONS - - # @@protoc_insertion_point(class_scope:google.protobuf.ServiceOptions) - -class MethodOptions(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _METHODOPTIONS - - # @@protoc_insertion_point(class_scope:google.protobuf.MethodOptions) - -class UninterpretedOption(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - - class NamePart(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _UNINTERPRETEDOPTION_NAMEPART - - # @@protoc_insertion_point(class_scope:google.protobuf.UninterpretedOption.NamePart) - DESCRIPTOR = _UNINTERPRETEDOPTION - - # @@protoc_insertion_point(class_scope:google.protobuf.UninterpretedOption) - -class SourceCodeInfo(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - - class Location(_message.Message): - __metaclass__ = _reflection.GeneratedProtocolMessageType - DESCRIPTOR = _SOURCECODEINFO_LOCATION - - # @@protoc_insertion_point(class_scope:google.protobuf.SourceCodeInfo.Location) - DESCRIPTOR = _SOURCECODEINFO - - # @@protoc_insertion_point(class_scope:google.protobuf.SourceCodeInfo) - - -# @@protoc_insertion_point(module_scope) diff --git a/generator/google/protobuf/descriptor_pool.py b/generator/google/protobuf/descriptor_pool.py deleted file mode 100644 index 8f1f445..0000000 --- a/generator/google/protobuf/descriptor_pool.py +++ /dev/null @@ -1,527 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Provides DescriptorPool to use as a container for proto2 descriptors. - -The DescriptorPool is used in conjection with a DescriptorDatabase to maintain -a collection of protocol buffer descriptors for use when dynamically creating -message types at runtime. - -For most applications protocol buffers should be used via modules generated by -the protocol buffer compiler tool. This should only be used when the type of -protocol buffers used in an application or library cannot be predetermined. - -Below is a straightforward example on how to use this class: - - pool = DescriptorPool() - file_descriptor_protos = [ ... ] - for file_descriptor_proto in file_descriptor_protos: - pool.Add(file_descriptor_proto) - my_message_descriptor = pool.FindMessageTypeByName('some.package.MessageType') - -The message descriptor can be used in conjunction with the message_factory -module in order to create a protocol buffer class that can be encoded and -decoded. -""" - -__author__ = 'matthewtoia@google.com (Matt Toia)' - -from google.protobuf import descriptor_pb2 -from google.protobuf import descriptor -from google.protobuf import descriptor_database - - -class DescriptorPool(object): - """A collection of protobufs dynamically constructed by descriptor protos.""" - - def __init__(self, descriptor_db=None): - """Initializes a Pool of proto buffs. - - The descriptor_db argument to the constructor is provided to allow - specialized file descriptor proto lookup code to be triggered on demand. An - example would be an implementation which will read and compile a file - specified in a call to FindFileByName() and not require the call to Add() - at all. Results from this database will be cached internally here as well. - - Args: - descriptor_db: A secondary source of file descriptors. - """ - - self._internal_db = descriptor_database.DescriptorDatabase() - self._descriptor_db = descriptor_db - self._descriptors = {} - self._enum_descriptors = {} - self._file_descriptors = {} - - def Add(self, file_desc_proto): - """Adds the FileDescriptorProto and its types to this pool. - - Args: - file_desc_proto: The FileDescriptorProto to add. - """ - - self._internal_db.Add(file_desc_proto) - - def FindFileByName(self, file_name): - """Gets a FileDescriptor by file name. - - Args: - file_name: The path to the file to get a descriptor for. - - Returns: - A FileDescriptor for the named file. - - Raises: - KeyError: if the file can not be found in the pool. - """ - - try: - file_proto = self._internal_db.FindFileByName(file_name) - except KeyError as error: - if self._descriptor_db: - file_proto = self._descriptor_db.FindFileByName(file_name) - else: - raise error - if not file_proto: - raise KeyError('Cannot find a file named %s' % file_name) - return self._ConvertFileProtoToFileDescriptor(file_proto) - - def FindFileContainingSymbol(self, symbol): - """Gets the FileDescriptor for the file containing the specified symbol. - - Args: - symbol: The name of the symbol to search for. - - Returns: - A FileDescriptor that contains the specified symbol. - - Raises: - KeyError: if the file can not be found in the pool. - """ - - try: - file_proto = self._internal_db.FindFileContainingSymbol(symbol) - except KeyError as error: - if self._descriptor_db: - file_proto = self._descriptor_db.FindFileContainingSymbol(symbol) - else: - raise error - if not file_proto: - raise KeyError('Cannot find a file containing %s' % symbol) - return self._ConvertFileProtoToFileDescriptor(file_proto) - - def FindMessageTypeByName(self, full_name): - """Loads the named descriptor from the pool. - - Args: - full_name: The full name of the descriptor to load. - - Returns: - The descriptor for the named type. - """ - - full_name = full_name.lstrip('.') # fix inconsistent qualified name formats - if full_name not in self._descriptors: - self.FindFileContainingSymbol(full_name) - return self._descriptors[full_name] - - def FindEnumTypeByName(self, full_name): - """Loads the named enum descriptor from the pool. - - Args: - full_name: The full name of the enum descriptor to load. - - Returns: - The enum descriptor for the named type. - """ - - full_name = full_name.lstrip('.') # fix inconsistent qualified name formats - if full_name not in self._enum_descriptors: - self.FindFileContainingSymbol(full_name) - return self._enum_descriptors[full_name] - - def _ConvertFileProtoToFileDescriptor(self, file_proto): - """Creates a FileDescriptor from a proto or returns a cached copy. - - This method also has the side effect of loading all the symbols found in - the file into the appropriate dictionaries in the pool. - - Args: - file_proto: The proto to convert. - - Returns: - A FileDescriptor matching the passed in proto. - """ - - if file_proto.name not in self._file_descriptors: - file_descriptor = descriptor.FileDescriptor( - name=file_proto.name, - package=file_proto.package, - options=file_proto.options, - serialized_pb=file_proto.SerializeToString()) - scope = {} - dependencies = list(self._GetDeps(file_proto)) - - for dependency in dependencies: - dep_desc = self.FindFileByName(dependency.name) - dep_proto = descriptor_pb2.FileDescriptorProto.FromString( - dep_desc.serialized_pb) - package = '.' + dep_proto.package - package_prefix = package + '.' - - def _strip_package(symbol): - if symbol.startswith(package_prefix): - return symbol[len(package_prefix):] - return symbol - - symbols = list(self._ExtractSymbols(dep_proto.message_type, package)) - scope.update(symbols) - scope.update((_strip_package(k), v) for k, v in symbols) - - symbols = list(self._ExtractEnums(dep_proto.enum_type, package)) - scope.update(symbols) - scope.update((_strip_package(k), v) for k, v in symbols) - - for message_type in file_proto.message_type: - message_desc = self._ConvertMessageDescriptor( - message_type, file_proto.package, file_descriptor, scope) - file_descriptor.message_types_by_name[message_desc.name] = message_desc - for enum_type in file_proto.enum_type: - self._ConvertEnumDescriptor(enum_type, file_proto.package, - file_descriptor, None, scope) - for desc_proto in self._ExtractMessages(file_proto.message_type): - self._SetFieldTypes(desc_proto, scope) - - for desc_proto in file_proto.message_type: - desc = scope[desc_proto.name] - file_descriptor.message_types_by_name[desc_proto.name] = desc - self.Add(file_proto) - self._file_descriptors[file_proto.name] = file_descriptor - - return self._file_descriptors[file_proto.name] - - def _ConvertMessageDescriptor(self, desc_proto, package=None, file_desc=None, - scope=None): - """Adds the proto to the pool in the specified package. - - Args: - desc_proto: The descriptor_pb2.DescriptorProto protobuf message. - package: The package the proto should be located in. - file_desc: The file containing this message. - scope: Dict mapping short and full symbols to message and enum types. - - Returns: - The added descriptor. - """ - - if package: - desc_name = '.'.join((package, desc_proto.name)) - else: - desc_name = desc_proto.name - - if file_desc is None: - file_name = None - else: - file_name = file_desc.name - - if scope is None: - scope = {} - - nested = [ - self._ConvertMessageDescriptor(nested, desc_name, file_desc, scope) - for nested in desc_proto.nested_type] - enums = [ - self._ConvertEnumDescriptor(enum, desc_name, file_desc, None, scope) - for enum in desc_proto.enum_type] - fields = [self._MakeFieldDescriptor(field, desc_name, index) - for index, field in enumerate(desc_proto.field)] - extensions = [self._MakeFieldDescriptor(extension, desc_name, True) - for index, extension in enumerate(desc_proto.extension)] - extension_ranges = [(r.start, r.end) for r in desc_proto.extension_range] - if extension_ranges: - is_extendable = True - else: - is_extendable = False - desc = descriptor.Descriptor( - name=desc_proto.name, - full_name=desc_name, - filename=file_name, - containing_type=None, - fields=fields, - nested_types=nested, - enum_types=enums, - extensions=extensions, - options=desc_proto.options, - is_extendable=is_extendable, - extension_ranges=extension_ranges, - file=file_desc, - serialized_start=None, - serialized_end=None) - for nested in desc.nested_types: - nested.containing_type = desc - for enum in desc.enum_types: - enum.containing_type = desc - scope[desc_proto.name] = desc - scope['.' + desc_name] = desc - self._descriptors[desc_name] = desc - return desc - - def _ConvertEnumDescriptor(self, enum_proto, package=None, file_desc=None, - containing_type=None, scope=None): - """Make a protobuf EnumDescriptor given an EnumDescriptorProto protobuf. - - Args: - enum_proto: The descriptor_pb2.EnumDescriptorProto protobuf message. - package: Optional package name for the new message EnumDescriptor. - file_desc: The file containing the enum descriptor. - containing_type: The type containing this enum. - scope: Scope containing available types. - - Returns: - The added descriptor - """ - - if package: - enum_name = '.'.join((package, enum_proto.name)) - else: - enum_name = enum_proto.name - - if file_desc is None: - file_name = None - else: - file_name = file_desc.name - - values = [self._MakeEnumValueDescriptor(value, index) - for index, value in enumerate(enum_proto.value)] - desc = descriptor.EnumDescriptor(name=enum_proto.name, - full_name=enum_name, - filename=file_name, - file=file_desc, - values=values, - containing_type=containing_type, - options=enum_proto.options) - scope[enum_proto.name] = desc - scope['.%s' % enum_name] = desc - self._enum_descriptors[enum_name] = desc - return desc - - def _MakeFieldDescriptor(self, field_proto, message_name, index, - is_extension=False): - """Creates a field descriptor from a FieldDescriptorProto. - - For message and enum type fields, this method will do a look up - in the pool for the appropriate descriptor for that type. If it - is unavailable, it will fall back to the _source function to - create it. If this type is still unavailable, construction will - fail. - - Args: - field_proto: The proto describing the field. - message_name: The name of the containing message. - index: Index of the field - is_extension: Indication that this field is for an extension. - - Returns: - An initialized FieldDescriptor object - """ - - if message_name: - full_name = '.'.join((message_name, field_proto.name)) - else: - full_name = field_proto.name - - return descriptor.FieldDescriptor( - name=field_proto.name, - full_name=full_name, - index=index, - number=field_proto.number, - type=field_proto.type, - cpp_type=None, - message_type=None, - enum_type=None, - containing_type=None, - label=field_proto.label, - has_default_value=False, - default_value=None, - is_extension=is_extension, - extension_scope=None, - options=field_proto.options) - - def _SetFieldTypes(self, desc_proto, scope): - """Sets the field's type, cpp_type, message_type and enum_type. - - Args: - desc_proto: The message descriptor to update. - scope: Enclosing scope of available types. - """ - - desc = scope[desc_proto.name] - for field_proto, field_desc in zip(desc_proto.field, desc.fields): - if field_proto.type_name: - type_name = field_proto.type_name - if type_name not in scope: - type_name = '.' + type_name - desc = scope[type_name] - else: - desc = None - - if not field_proto.HasField('type'): - if isinstance(desc, descriptor.Descriptor): - field_proto.type = descriptor.FieldDescriptor.TYPE_MESSAGE - else: - field_proto.type = descriptor.FieldDescriptor.TYPE_ENUM - - field_desc.cpp_type = descriptor.FieldDescriptor.ProtoTypeToCppProtoType( - field_proto.type) - - if (field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE - or field_proto.type == descriptor.FieldDescriptor.TYPE_GROUP): - field_desc.message_type = desc - - if field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM: - field_desc.enum_type = desc - - if field_proto.label == descriptor.FieldDescriptor.LABEL_REPEATED: - field_desc.has_default = False - field_desc.default_value = [] - elif field_proto.HasField('default_value'): - field_desc.has_default = True - if (field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE or - field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT): - field_desc.default_value = float(field_proto.default_value) - elif field_proto.type == descriptor.FieldDescriptor.TYPE_STRING: - field_desc.default_value = field_proto.default_value - elif field_proto.type == descriptor.FieldDescriptor.TYPE_BOOL: - field_desc.default_value = field_proto.default_value.lower() == 'true' - elif field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM: - field_desc.default_value = field_desc.enum_type.values_by_name[ - field_proto.default_value].index - else: - field_desc.default_value = int(field_proto.default_value) - else: - field_desc.has_default = False - field_desc.default_value = None - - field_desc.type = field_proto.type - - for nested_type in desc_proto.nested_type: - self._SetFieldTypes(nested_type, scope) - - def _MakeEnumValueDescriptor(self, value_proto, index): - """Creates a enum value descriptor object from a enum value proto. - - Args: - value_proto: The proto describing the enum value. - index: The index of the enum value. - - Returns: - An initialized EnumValueDescriptor object. - """ - - return descriptor.EnumValueDescriptor( - name=value_proto.name, - index=index, - number=value_proto.number, - options=value_proto.options, - type=None) - - def _ExtractSymbols(self, desc_protos, package): - """Pulls out all the symbols from descriptor protos. - - Args: - desc_protos: The protos to extract symbols from. - package: The package containing the descriptor type. - Yields: - A two element tuple of the type name and descriptor object. - """ - - for desc_proto in desc_protos: - if package: - message_name = '.'.join((package, desc_proto.name)) - else: - message_name = desc_proto.name - message_desc = self.FindMessageTypeByName(message_name) - yield (message_name, message_desc) - for symbol in self._ExtractSymbols(desc_proto.nested_type, message_name): - yield symbol - for symbol in self._ExtractEnums(desc_proto.enum_type, message_name): - yield symbol - - def _ExtractEnums(self, enum_protos, package): - """Pulls out all the symbols from enum protos. - - Args: - enum_protos: The protos to extract symbols from. - package: The package containing the enum type. - - Yields: - A two element tuple of the type name and enum descriptor object. - """ - - for enum_proto in enum_protos: - if package: - enum_name = '.'.join((package, enum_proto.name)) - else: - enum_name = enum_proto.name - enum_desc = self.FindEnumTypeByName(enum_name) - yield (enum_name, enum_desc) - - def _ExtractMessages(self, desc_protos): - """Pulls out all the message protos from descriptos. - - Args: - desc_protos: The protos to extract symbols from. - - Yields: - Descriptor protos. - """ - - for desc_proto in desc_protos: - yield desc_proto - for message in self._ExtractMessages(desc_proto.nested_type): - yield message - - def _GetDeps(self, file_proto): - """Recursively finds dependencies for file protos. - - Args: - file_proto: The proto to get dependencies from. - - Yields: - Each direct and indirect dependency. - """ - - for dependency in file_proto.dependency: - dep_desc = self.FindFileByName(dependency) - dep_proto = descriptor_pb2.FileDescriptorProto.FromString( - dep_desc.serialized_pb) - yield dep_proto - for parent_dep in self._GetDeps(dep_proto): - yield parent_dep diff --git a/generator/google/protobuf/internal/__init__.py b/generator/google/protobuf/internal/__init__.py deleted file mode 100644 index e69de29..0000000 --- a/generator/google/protobuf/internal/__init__.py +++ /dev/null diff --git a/generator/google/protobuf/internal/api_implementation.py b/generator/google/protobuf/internal/api_implementation.py deleted file mode 100644 index ce02a32..0000000 --- a/generator/google/protobuf/internal/api_implementation.py +++ /dev/null @@ -1,87 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -""" -This module is the central entity that determines which implementation of the -API is used. -""" - -__author__ = 'petar@google.com (Petar Petrov)' - -import os -# This environment variable can be used to switch to a certain implementation -# of the Python API. Right now only 'python' and 'cpp' are valid values. Any -# other value will be ignored. -_implementation_type = os.getenv('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION', - 'python') - - -if _implementation_type != 'python': - # For now, by default use the pure-Python implementation. - # The code below checks if the C extension is available and - # uses it if it is available. - _implementation_type = 'cpp' - ## Determine automatically which implementation to use. - #try: - # from google.protobuf.internal import cpp_message - # _implementation_type = 'cpp' - #except ImportError, e: - # _implementation_type = 'python' - - -# This environment variable can be used to switch between the two -# 'cpp' implementations. Right now only 1 and 2 are valid values. Any -# other value will be ignored. -_implementation_version_str = os.getenv( - 'PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION', - '1') - - -if _implementation_version_str not in ('1', '2'): - raise ValueError( - "unsupported PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION_VERSION: '" + - _implementation_version_str + "' (supported versions: 1, 2)" - ) - - -_implementation_version = int(_implementation_version_str) - - - -# Usage of this function is discouraged. Clients shouldn't care which -# implementation of the API is in use. Note that there is no guarantee -# that differences between APIs will be maintained. -# Please don't use this function if possible. -def Type(): - return _implementation_type - -# See comment on 'Type' above. -def Version(): - return _implementation_version diff --git a/generator/google/protobuf/internal/containers.py b/generator/google/protobuf/internal/containers.py deleted file mode 100644 index 34b35f8..0000000 --- a/generator/google/protobuf/internal/containers.py +++ /dev/null @@ -1,269 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Contains container classes to represent different protocol buffer types. - -This file defines container classes which represent categories of protocol -buffer field types which need extra maintenance. Currently these categories -are: - - Repeated scalar fields - These are all repeated fields which aren't - composite (e.g. they are of simple types like int32, string, etc). - - Repeated composite fields - Repeated fields which are composite. This - includes groups and nested messages. -""" - -__author__ = 'petar@google.com (Petar Petrov)' - - -class BaseContainer(object): - - """Base container class.""" - - # Minimizes memory usage and disallows assignment to other attributes. - __slots__ = ['_message_listener', '_values'] - - def __init__(self, message_listener): - """ - Args: - message_listener: A MessageListener implementation. - The RepeatedScalarFieldContainer will call this object's - Modified() method when it is modified. - """ - self._message_listener = message_listener - self._values = [] - - def __getitem__(self, key): - """Retrieves item by the specified key.""" - return self._values[key] - - def __len__(self): - """Returns the number of elements in the container.""" - return len(self._values) - - def __ne__(self, other): - """Checks if another instance isn't equal to this one.""" - # The concrete classes should define __eq__. - return not self == other - - def __hash__(self): - raise TypeError('unhashable object') - - def __repr__(self): - return repr(self._values) - - def sort(self, *args, **kwargs): - # Continue to support the old sort_function keyword argument. - # This is expected to be a rare occurrence, so use LBYL to avoid - # the overhead of actually catching KeyError. - if 'sort_function' in kwargs: - kwargs['cmp'] = kwargs.pop('sort_function') - self._values.sort(*args, **kwargs) - - -class RepeatedScalarFieldContainer(BaseContainer): - - """Simple, type-checked, list-like container for holding repeated scalars.""" - - # Disallows assignment to other attributes. - __slots__ = ['_type_checker'] - - def __init__(self, message_listener, type_checker): - """ - Args: - message_listener: A MessageListener implementation. - The RepeatedScalarFieldContainer will call this object's - Modified() method when it is modified. - type_checker: A type_checkers.ValueChecker instance to run on elements - inserted into this container. - """ - super(RepeatedScalarFieldContainer, self).__init__(message_listener) - self._type_checker = type_checker - - def append(self, value): - """Appends an item to the list. Similar to list.append().""" - self._type_checker.CheckValue(value) - self._values.append(value) - if not self._message_listener.dirty: - self._message_listener.Modified() - - def insert(self, key, value): - """Inserts the item at the specified position. Similar to list.insert().""" - self._type_checker.CheckValue(value) - self._values.insert(key, value) - if not self._message_listener.dirty: - self._message_listener.Modified() - - def extend(self, elem_seq): - """Extends by appending the given sequence. Similar to list.extend().""" - if not elem_seq: - return - - new_values = [] - for elem in elem_seq: - self._type_checker.CheckValue(elem) - new_values.append(elem) - self._values.extend(new_values) - self._message_listener.Modified() - - def MergeFrom(self, other): - """Appends the contents of another repeated field of the same type to this - one. We do not check the types of the individual fields. - """ - self._values.extend(other._values) - self._message_listener.Modified() - - def remove(self, elem): - """Removes an item from the list. Similar to list.remove().""" - self._values.remove(elem) - self._message_listener.Modified() - - def __setitem__(self, key, value): - """Sets the item on the specified position.""" - self._type_checker.CheckValue(value) - self._values[key] = value - self._message_listener.Modified() - - def __getslice__(self, start, stop): - """Retrieves the subset of items from between the specified indices.""" - return self._values[start:stop] - - def __setslice__(self, start, stop, values): - """Sets the subset of items from between the specified indices.""" - new_values = [] - for value in values: - self._type_checker.CheckValue(value) - new_values.append(value) - self._values[start:stop] = new_values - self._message_listener.Modified() - - def __delitem__(self, key): - """Deletes the item at the specified position.""" - del self._values[key] - self._message_listener.Modified() - - def __delslice__(self, start, stop): - """Deletes the subset of items from between the specified indices.""" - del self._values[start:stop] - self._message_listener.Modified() - - def __eq__(self, other): - """Compares the current instance with another one.""" - if self is other: - return True - # Special case for the same type which should be common and fast. - if isinstance(other, self.__class__): - return other._values == self._values - # We are presumably comparing against some other sequence type. - return other == self._values - - -class RepeatedCompositeFieldContainer(BaseContainer): - - """Simple, list-like container for holding repeated composite fields.""" - - # Disallows assignment to other attributes. - __slots__ = ['_message_descriptor'] - - def __init__(self, message_listener, message_descriptor): - """ - Note that we pass in a descriptor instead of the generated directly, - since at the time we construct a _RepeatedCompositeFieldContainer we - haven't yet necessarily initialized the type that will be contained in the - container. - - Args: - message_listener: A MessageListener implementation. - The RepeatedCompositeFieldContainer will call this object's - Modified() method when it is modified. - message_descriptor: A Descriptor instance describing the protocol type - that should be present in this container. We'll use the - _concrete_class field of this descriptor when the client calls add(). - """ - super(RepeatedCompositeFieldContainer, self).__init__(message_listener) - self._message_descriptor = message_descriptor - - def add(self, **kwargs): - """Adds a new element at the end of the list and returns it. Keyword - arguments may be used to initialize the element. - """ - new_element = self._message_descriptor._concrete_class(**kwargs) - new_element._SetListener(self._message_listener) - self._values.append(new_element) - if not self._message_listener.dirty: - self._message_listener.Modified() - return new_element - - def extend(self, elem_seq): - """Extends by appending the given sequence of elements of the same type - as this one, copying each individual message. - """ - message_class = self._message_descriptor._concrete_class - listener = self._message_listener - values = self._values - for message in elem_seq: - new_element = message_class() - new_element._SetListener(listener) - new_element.MergeFrom(message) - values.append(new_element) - listener.Modified() - - def MergeFrom(self, other): - """Appends the contents of another repeated field of the same type to this - one, copying each individual message. - """ - self.extend(other._values) - - def remove(self, elem): - """Removes an item from the list. Similar to list.remove().""" - self._values.remove(elem) - self._message_listener.Modified() - - def __getslice__(self, start, stop): - """Retrieves the subset of items from between the specified indices.""" - return self._values[start:stop] - - def __delitem__(self, key): - """Deletes the item at the specified position.""" - del self._values[key] - self._message_listener.Modified() - - def __delslice__(self, start, stop): - """Deletes the subset of items from between the specified indices.""" - del self._values[start:stop] - self._message_listener.Modified() - - def __eq__(self, other): - """Compares the current instance with another one.""" - if self is other: - return True - if not isinstance(other, self.__class__): - raise TypeError('Can only compare repeated composite fields against ' - 'other repeated composite fields.') - return self._values == other._values diff --git a/generator/google/protobuf/internal/decoder.py b/generator/google/protobuf/internal/decoder.py deleted file mode 100644 index cb6f572..0000000 --- a/generator/google/protobuf/internal/decoder.py +++ /dev/null @@ -1,720 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Code for decoding protocol buffer primitives. - -This code is very similar to encoder.py -- read the docs for that module first. - -A "decoder" is a function with the signature: - Decode(buffer, pos, end, message, field_dict) -The arguments are: - buffer: The string containing the encoded message. - pos: The current position in the string. - end: The position in the string where the current message ends. May be - less than len(buffer) if we're reading a sub-message. - message: The message object into which we're parsing. - field_dict: message._fields (avoids a hashtable lookup). -The decoder reads the field and stores it into field_dict, returning the new -buffer position. A decoder for a repeated field may proactively decode all of -the elements of that field, if they appear consecutively. - -Note that decoders may throw any of the following: - IndexError: Indicates a truncated message. - struct.error: Unpacking of a fixed-width field failed. - message.DecodeError: Other errors. - -Decoders are expected to raise an exception if they are called with pos > end. -This allows callers to be lax about bounds checking: it's fineto read past -"end" as long as you are sure that someone else will notice and throw an -exception later on. - -Something up the call stack is expected to catch IndexError and struct.error -and convert them to message.DecodeError. - -Decoders are constructed using decoder constructors with the signature: - MakeDecoder(field_number, is_repeated, is_packed, key, new_default) -The arguments are: - field_number: The field number of the field we want to decode. - is_repeated: Is the field a repeated field? (bool) - is_packed: Is the field a packed field? (bool) - key: The key to use when looking up the field within field_dict. - (This is actually the FieldDescriptor but nothing in this - file should depend on that.) - new_default: A function which takes a message object as a parameter and - returns a new instance of the default value for this field. - (This is called for repeated fields and sub-messages, when an - instance does not already exist.) - -As with encoders, we define a decoder constructor for every type of field. -Then, for every field of every message class we construct an actual decoder. -That decoder goes into a dict indexed by tag, so when we decode a message -we repeatedly read a tag, look up the corresponding decoder, and invoke it. -""" - -__author__ = 'kenton@google.com (Kenton Varda)' - -import struct -from google.protobuf.internal import encoder -from google.protobuf.internal import wire_format -from google.protobuf import message - - -# This will overflow and thus become IEEE-754 "infinity". We would use -# "float('inf')" but it doesn't work on Windows pre-Python-2.6. -_POS_INF = 1e10000 -_NEG_INF = -_POS_INF -_NAN = _POS_INF * 0 - - -# This is not for optimization, but rather to avoid conflicts with local -# variables named "message". -_DecodeError = message.DecodeError - - -def _VarintDecoder(mask): - """Return an encoder for a basic varint value (does not include tag). - - Decoded values will be bitwise-anded with the given mask before being - returned, e.g. to limit them to 32 bits. The returned decoder does not - take the usual "end" parameter -- the caller is expected to do bounds checking - after the fact (often the caller can defer such checking until later). The - decoder returns a (value, new_pos) pair. - """ - - local_ord = ord - def DecodeVarint(buffer, pos): - result = 0 - shift = 0 - while 1: - b = local_ord(buffer[pos]) - result |= ((b & 0x7f) << shift) - pos += 1 - if not (b & 0x80): - result &= mask - return (result, pos) - shift += 7 - if shift >= 64: - raise _DecodeError('Too many bytes when decoding varint.') - return DecodeVarint - - -def _SignedVarintDecoder(mask): - """Like _VarintDecoder() but decodes signed values.""" - - local_ord = ord - def DecodeVarint(buffer, pos): - result = 0 - shift = 0 - while 1: - b = local_ord(buffer[pos]) - result |= ((b & 0x7f) << shift) - pos += 1 - if not (b & 0x80): - if result > 0x7fffffffffffffff: - result -= (1 << 64) - result |= ~mask - else: - result &= mask - return (result, pos) - shift += 7 - if shift >= 64: - raise _DecodeError('Too many bytes when decoding varint.') - return DecodeVarint - - -_DecodeVarint = _VarintDecoder((1 << 64) - 1) -_DecodeSignedVarint = _SignedVarintDecoder((1 << 64) - 1) - -# Use these versions for values which must be limited to 32 bits. -_DecodeVarint32 = _VarintDecoder((1 << 32) - 1) -_DecodeSignedVarint32 = _SignedVarintDecoder((1 << 32) - 1) - - -def ReadTag(buffer, pos): - """Read a tag from the buffer, and return a (tag_bytes, new_pos) tuple. - - We return the raw bytes of the tag rather than decoding them. The raw - bytes can then be used to look up the proper decoder. This effectively allows - us to trade some work that would be done in pure-python (decoding a varint) - for work that is done in C (searching for a byte string in a hash table). - In a low-level language it would be much cheaper to decode the varint and - use that, but not in Python. - """ - - start = pos - while ord(buffer[pos]) & 0x80: - pos += 1 - pos += 1 - return (buffer[start:pos], pos) - - -# -------------------------------------------------------------------- - - -def _SimpleDecoder(wire_type, decode_value): - """Return a constructor for a decoder for fields of a particular type. - - Args: - wire_type: The field's wire type. - decode_value: A function which decodes an individual value, e.g. - _DecodeVarint() - """ - - def SpecificDecoder(field_number, is_repeated, is_packed, key, new_default): - if is_packed: - local_DecodeVarint = _DecodeVarint - def DecodePackedField(buffer, pos, end, message, field_dict): - value = field_dict.get(key) - if value is None: - value = field_dict.setdefault(key, new_default(message)) - (endpoint, pos) = local_DecodeVarint(buffer, pos) - endpoint += pos - if endpoint > end: - raise _DecodeError('Truncated message.') - while pos < endpoint: - (element, pos) = decode_value(buffer, pos) - value.append(element) - if pos > endpoint: - del value[-1] # Discard corrupt value. - raise _DecodeError('Packed element was truncated.') - return pos - return DecodePackedField - elif is_repeated: - tag_bytes = encoder.TagBytes(field_number, wire_type) - tag_len = len(tag_bytes) - def DecodeRepeatedField(buffer, pos, end, message, field_dict): - value = field_dict.get(key) - if value is None: - value = field_dict.setdefault(key, new_default(message)) - while 1: - (element, new_pos) = decode_value(buffer, pos) - value.append(element) - # Predict that the next tag is another copy of the same repeated - # field. - pos = new_pos + tag_len - if buffer[new_pos:pos] != tag_bytes or new_pos >= end: - # Prediction failed. Return. - if new_pos > end: - raise _DecodeError('Truncated message.') - return new_pos - return DecodeRepeatedField - else: - def DecodeField(buffer, pos, end, message, field_dict): - (field_dict[key], pos) = decode_value(buffer, pos) - if pos > end: - del field_dict[key] # Discard corrupt value. - raise _DecodeError('Truncated message.') - return pos - return DecodeField - - return SpecificDecoder - - -def _ModifiedDecoder(wire_type, decode_value, modify_value): - """Like SimpleDecoder but additionally invokes modify_value on every value - before storing it. Usually modify_value is ZigZagDecode. - """ - - # Reusing _SimpleDecoder is slightly slower than copying a bunch of code, but - # not enough to make a significant difference. - - def InnerDecode(buffer, pos): - (result, new_pos) = decode_value(buffer, pos) - return (modify_value(result), new_pos) - return _SimpleDecoder(wire_type, InnerDecode) - - -def _StructPackDecoder(wire_type, format): - """Return a constructor for a decoder for a fixed-width field. - - Args: - wire_type: The field's wire type. - format: The format string to pass to struct.unpack(). - """ - - value_size = struct.calcsize(format) - local_unpack = struct.unpack - - # Reusing _SimpleDecoder is slightly slower than copying a bunch of code, but - # not enough to make a significant difference. - - # Note that we expect someone up-stack to catch struct.error and convert - # it to _DecodeError -- this way we don't have to set up exception- - # handling blocks every time we parse one value. - - def InnerDecode(buffer, pos): - new_pos = pos + value_size - result = local_unpack(format, buffer[pos:new_pos])[0] - return (result, new_pos) - return _SimpleDecoder(wire_type, InnerDecode) - - -def _FloatDecoder(): - """Returns a decoder for a float field. - - This code works around a bug in struct.unpack for non-finite 32-bit - floating-point values. - """ - - local_unpack = struct.unpack - - def InnerDecode(buffer, pos): - # We expect a 32-bit value in little-endian byte order. Bit 1 is the sign - # bit, bits 2-9 represent the exponent, and bits 10-32 are the significand. - new_pos = pos + 4 - float_bytes = buffer[pos:new_pos] - - # If this value has all its exponent bits set, then it's non-finite. - # In Python 2.4, struct.unpack will convert it to a finite 64-bit value. - # To avoid that, we parse it specially. - if ((float_bytes[3] in '\x7F\xFF') - and (float_bytes[2] >= '\x80')): - # If at least one significand bit is set... - if float_bytes[0:3] != '\x00\x00\x80': - return (_NAN, new_pos) - # If sign bit is set... - if float_bytes[3] == '\xFF': - return (_NEG_INF, new_pos) - return (_POS_INF, new_pos) - - # Note that we expect someone up-stack to catch struct.error and convert - # it to _DecodeError -- this way we don't have to set up exception- - # handling blocks every time we parse one value. - result = local_unpack('<f', float_bytes)[0] - return (result, new_pos) - return _SimpleDecoder(wire_format.WIRETYPE_FIXED32, InnerDecode) - - -def _DoubleDecoder(): - """Returns a decoder for a double field. - - This code works around a bug in struct.unpack for not-a-number. - """ - - local_unpack = struct.unpack - - def InnerDecode(buffer, pos): - # We expect a 64-bit value in little-endian byte order. Bit 1 is the sign - # bit, bits 2-12 represent the exponent, and bits 13-64 are the significand. - new_pos = pos + 8 - double_bytes = buffer[pos:new_pos] - - # If this value has all its exponent bits set and at least one significand - # bit set, it's not a number. In Python 2.4, struct.unpack will treat it - # as inf or -inf. To avoid that, we treat it specially. - if ((double_bytes[7] in '\x7F\xFF') - and (double_bytes[6] >= '\xF0') - and (double_bytes[0:7] != '\x00\x00\x00\x00\x00\x00\xF0')): - return (_NAN, new_pos) - - # Note that we expect someone up-stack to catch struct.error and convert - # it to _DecodeError -- this way we don't have to set up exception- - # handling blocks every time we parse one value. - result = local_unpack('<d', double_bytes)[0] - return (result, new_pos) - return _SimpleDecoder(wire_format.WIRETYPE_FIXED64, InnerDecode) - - -# -------------------------------------------------------------------- - - -Int32Decoder = EnumDecoder = _SimpleDecoder( - wire_format.WIRETYPE_VARINT, _DecodeSignedVarint32) - -Int64Decoder = _SimpleDecoder( - wire_format.WIRETYPE_VARINT, _DecodeSignedVarint) - -UInt32Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint32) -UInt64Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint) - -SInt32Decoder = _ModifiedDecoder( - wire_format.WIRETYPE_VARINT, _DecodeVarint32, wire_format.ZigZagDecode) -SInt64Decoder = _ModifiedDecoder( - wire_format.WIRETYPE_VARINT, _DecodeVarint, wire_format.ZigZagDecode) - -# Note that Python conveniently guarantees that when using the '<' prefix on -# formats, they will also have the same size across all platforms (as opposed -# to without the prefix, where their sizes depend on the C compiler's basic -# type sizes). -Fixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, '<I') -Fixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, '<Q') -SFixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, '<i') -SFixed64Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED64, '<q') -FloatDecoder = _FloatDecoder() -DoubleDecoder = _DoubleDecoder() - -BoolDecoder = _ModifiedDecoder( - wire_format.WIRETYPE_VARINT, _DecodeVarint, bool) - - -def StringDecoder(field_number, is_repeated, is_packed, key, new_default): - """Returns a decoder for a string field.""" - - local_DecodeVarint = _DecodeVarint - local_unicode = unicode - - assert not is_packed - if is_repeated: - tag_bytes = encoder.TagBytes(field_number, - wire_format.WIRETYPE_LENGTH_DELIMITED) - tag_len = len(tag_bytes) - def DecodeRepeatedField(buffer, pos, end, message, field_dict): - value = field_dict.get(key) - if value is None: - value = field_dict.setdefault(key, new_default(message)) - while 1: - (size, pos) = local_DecodeVarint(buffer, pos) - new_pos = pos + size - if new_pos > end: - raise _DecodeError('Truncated string.') - value.append(local_unicode(buffer[pos:new_pos], 'utf-8')) - # Predict that the next tag is another copy of the same repeated field. - pos = new_pos + tag_len - if buffer[new_pos:pos] != tag_bytes or new_pos == end: - # Prediction failed. Return. - return new_pos - return DecodeRepeatedField - else: - def DecodeField(buffer, pos, end, message, field_dict): - (size, pos) = local_DecodeVarint(buffer, pos) - new_pos = pos + size - if new_pos > end: - raise _DecodeError('Truncated string.') - field_dict[key] = local_unicode(buffer[pos:new_pos], 'utf-8') - return new_pos - return DecodeField - - -def BytesDecoder(field_number, is_repeated, is_packed, key, new_default): - """Returns a decoder for a bytes field.""" - - local_DecodeVarint = _DecodeVarint - - assert not is_packed - if is_repeated: - tag_bytes = encoder.TagBytes(field_number, - wire_format.WIRETYPE_LENGTH_DELIMITED) - tag_len = len(tag_bytes) - def DecodeRepeatedField(buffer, pos, end, message, field_dict): - value = field_dict.get(key) - if value is None: - value = field_dict.setdefault(key, new_default(message)) - while 1: - (size, pos) = local_DecodeVarint(buffer, pos) - new_pos = pos + size - if new_pos > end: - raise _DecodeError('Truncated string.') - value.append(buffer[pos:new_pos]) - # Predict that the next tag is another copy of the same repeated field. - pos = new_pos + tag_len - if buffer[new_pos:pos] != tag_bytes or new_pos == end: - # Prediction failed. Return. - return new_pos - return DecodeRepeatedField - else: - def DecodeField(buffer, pos, end, message, field_dict): - (size, pos) = local_DecodeVarint(buffer, pos) - new_pos = pos + size - if new_pos > end: - raise _DecodeError('Truncated string.') - field_dict[key] = buffer[pos:new_pos] - return new_pos - return DecodeField - - -def GroupDecoder(field_number, is_repeated, is_packed, key, new_default): - """Returns a decoder for a group field.""" - - end_tag_bytes = encoder.TagBytes(field_number, - wire_format.WIRETYPE_END_GROUP) - end_tag_len = len(end_tag_bytes) - - assert not is_packed - if is_repeated: - tag_bytes = encoder.TagBytes(field_number, - wire_format.WIRETYPE_START_GROUP) - tag_len = len(tag_bytes) - def DecodeRepeatedField(buffer, pos, end, message, field_dict): - value = field_dict.get(key) - if value is None: - value = field_dict.setdefault(key, new_default(message)) - while 1: - value = field_dict.get(key) - if value is None: - value = field_dict.setdefault(key, new_default(message)) - # Read sub-message. - pos = value.add()._InternalParse(buffer, pos, end) - # Read end tag. - new_pos = pos+end_tag_len - if buffer[pos:new_pos] != end_tag_bytes or new_pos > end: - raise _DecodeError('Missing group end tag.') - # Predict that the next tag is another copy of the same repeated field. - pos = new_pos + tag_len - if buffer[new_pos:pos] != tag_bytes or new_pos == end: - # Prediction failed. Return. - return new_pos - return DecodeRepeatedField - else: - def DecodeField(buffer, pos, end, message, field_dict): - value = field_dict.get(key) - if value is None: - value = field_dict.setdefault(key, new_default(message)) - # Read sub-message. - pos = value._InternalParse(buffer, pos, end) - # Read end tag. - new_pos = pos+end_tag_len - if buffer[pos:new_pos] != end_tag_bytes or new_pos > end: - raise _DecodeError('Missing group end tag.') - return new_pos - return DecodeField - - -def MessageDecoder(field_number, is_repeated, is_packed, key, new_default): - """Returns a decoder for a message field.""" - - local_DecodeVarint = _DecodeVarint - - assert not is_packed - if is_repeated: - tag_bytes = encoder.TagBytes(field_number, - wire_format.WIRETYPE_LENGTH_DELIMITED) - tag_len = len(tag_bytes) - def DecodeRepeatedField(buffer, pos, end, message, field_dict): - value = field_dict.get(key) - if value is None: - value = field_dict.setdefault(key, new_default(message)) - while 1: - value = field_dict.get(key) - if value is None: - value = field_dict.setdefault(key, new_default(message)) - # Read length. - (size, pos) = local_DecodeVarint(buffer, pos) - new_pos = pos + size - if new_pos > end: - raise _DecodeError('Truncated message.') - # Read sub-message. - if value.add()._InternalParse(buffer, pos, new_pos) != new_pos: - # The only reason _InternalParse would return early is if it - # encountered an end-group tag. - raise _DecodeError('Unexpected end-group tag.') - # Predict that the next tag is another copy of the same repeated field. - pos = new_pos + tag_len - if buffer[new_pos:pos] != tag_bytes or new_pos == end: - # Prediction failed. Return. - return new_pos - return DecodeRepeatedField - else: - def DecodeField(buffer, pos, end, message, field_dict): - value = field_dict.get(key) - if value is None: - value = field_dict.setdefault(key, new_default(message)) - # Read length. - (size, pos) = local_DecodeVarint(buffer, pos) - new_pos = pos + size - if new_pos > end: - raise _DecodeError('Truncated message.') - # Read sub-message. - if value._InternalParse(buffer, pos, new_pos) != new_pos: - # The only reason _InternalParse would return early is if it encountered - # an end-group tag. - raise _DecodeError('Unexpected end-group tag.') - return new_pos - return DecodeField - - -# -------------------------------------------------------------------- - -MESSAGE_SET_ITEM_TAG = encoder.TagBytes(1, wire_format.WIRETYPE_START_GROUP) - -def MessageSetItemDecoder(extensions_by_number): - """Returns a decoder for a MessageSet item. - - The parameter is the _extensions_by_number map for the message class. - - The message set message looks like this: - message MessageSet { - repeated group Item = 1 { - required int32 type_id = 2; - required string message = 3; - } - } - """ - - type_id_tag_bytes = encoder.TagBytes(2, wire_format.WIRETYPE_VARINT) - message_tag_bytes = encoder.TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED) - item_end_tag_bytes = encoder.TagBytes(1, wire_format.WIRETYPE_END_GROUP) - - local_ReadTag = ReadTag - local_DecodeVarint = _DecodeVarint - local_SkipField = SkipField - - def DecodeItem(buffer, pos, end, message, field_dict): - message_set_item_start = pos - type_id = -1 - message_start = -1 - message_end = -1 - - # Technically, type_id and message can appear in any order, so we need - # a little loop here. - while 1: - (tag_bytes, pos) = local_ReadTag(buffer, pos) - if tag_bytes == type_id_tag_bytes: - (type_id, pos) = local_DecodeVarint(buffer, pos) - elif tag_bytes == message_tag_bytes: - (size, message_start) = local_DecodeVarint(buffer, pos) - pos = message_end = message_start + size - elif tag_bytes == item_end_tag_bytes: - break - else: - pos = SkipField(buffer, pos, end, tag_bytes) - if pos == -1: - raise _DecodeError('Missing group end tag.') - - if pos > end: - raise _DecodeError('Truncated message.') - - if type_id == -1: - raise _DecodeError('MessageSet item missing type_id.') - if message_start == -1: - raise _DecodeError('MessageSet item missing message.') - - extension = extensions_by_number.get(type_id) - if extension is not None: - value = field_dict.get(extension) - if value is None: - value = field_dict.setdefault( - extension, extension.message_type._concrete_class()) - if value._InternalParse(buffer, message_start,message_end) != message_end: - # The only reason _InternalParse would return early is if it encountered - # an end-group tag. - raise _DecodeError('Unexpected end-group tag.') - else: - if not message._unknown_fields: - message._unknown_fields = [] - message._unknown_fields.append((MESSAGE_SET_ITEM_TAG, - buffer[message_set_item_start:pos])) - - return pos - - return DecodeItem - -# -------------------------------------------------------------------- -# Optimization is not as heavy here because calls to SkipField() are rare, -# except for handling end-group tags. - -def _SkipVarint(buffer, pos, end): - """Skip a varint value. Returns the new position.""" - - while ord(buffer[pos]) & 0x80: - pos += 1 - pos += 1 - if pos > end: - raise _DecodeError('Truncated message.') - return pos - -def _SkipFixed64(buffer, pos, end): - """Skip a fixed64 value. Returns the new position.""" - - pos += 8 - if pos > end: - raise _DecodeError('Truncated message.') - return pos - -def _SkipLengthDelimited(buffer, pos, end): - """Skip a length-delimited value. Returns the new position.""" - - (size, pos) = _DecodeVarint(buffer, pos) - pos += size - if pos > end: - raise _DecodeError('Truncated message.') - return pos - -def _SkipGroup(buffer, pos, end): - """Skip sub-group. Returns the new position.""" - - while 1: - (tag_bytes, pos) = ReadTag(buffer, pos) - new_pos = SkipField(buffer, pos, end, tag_bytes) - if new_pos == -1: - return pos - pos = new_pos - -def _EndGroup(buffer, pos, end): - """Skipping an END_GROUP tag returns -1 to tell the parent loop to break.""" - - return -1 - -def _SkipFixed32(buffer, pos, end): - """Skip a fixed32 value. Returns the new position.""" - - pos += 4 - if pos > end: - raise _DecodeError('Truncated message.') - return pos - -def _RaiseInvalidWireType(buffer, pos, end): - """Skip function for unknown wire types. Raises an exception.""" - - raise _DecodeError('Tag had invalid wire type.') - -def _FieldSkipper(): - """Constructs the SkipField function.""" - - WIRETYPE_TO_SKIPPER = [ - _SkipVarint, - _SkipFixed64, - _SkipLengthDelimited, - _SkipGroup, - _EndGroup, - _SkipFixed32, - _RaiseInvalidWireType, - _RaiseInvalidWireType, - ] - - wiretype_mask = wire_format.TAG_TYPE_MASK - local_ord = ord - - def SkipField(buffer, pos, end, tag_bytes): - """Skips a field with the specified tag. - - |pos| should point to the byte immediately after the tag. - - Returns: - The new position (after the tag value), or -1 if the tag is an end-group - tag (in which case the calling loop should break). - """ - - # The wire type is always in the first byte since varints are little-endian. - wire_type = local_ord(tag_bytes[0]) & wiretype_mask - return WIRETYPE_TO_SKIPPER[wire_type](buffer, pos, end) - - return SkipField - -SkipField = _FieldSkipper() diff --git a/generator/google/protobuf/internal/encoder.py b/generator/google/protobuf/internal/encoder.py deleted file mode 100644 index 777975e..0000000 --- a/generator/google/protobuf/internal/encoder.py +++ /dev/null @@ -1,769 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Code for encoding protocol message primitives. - -Contains the logic for encoding every logical protocol field type -into one of the 5 physical wire types. - -This code is designed to push the Python interpreter's performance to the -limits. - -The basic idea is that at startup time, for every field (i.e. every -FieldDescriptor) we construct two functions: a "sizer" and an "encoder". The -sizer takes a value of this field's type and computes its byte size. The -encoder takes a writer function and a value. It encodes the value into byte -strings and invokes the writer function to write those strings. Typically the -writer function is the write() method of a cStringIO. - -We try to do as much work as possible when constructing the writer and the -sizer rather than when calling them. In particular: -* We copy any needed global functions to local variables, so that we do not need - to do costly global table lookups at runtime. -* Similarly, we try to do any attribute lookups at startup time if possible. -* Every field's tag is encoded to bytes at startup, since it can't change at - runtime. -* Whatever component of the field size we can compute at startup, we do. -* We *avoid* sharing code if doing so would make the code slower and not sharing - does not burden us too much. For example, encoders for repeated fields do - not just call the encoders for singular fields in a loop because this would - add an extra function call overhead for every loop iteration; instead, we - manually inline the single-value encoder into the loop. -* If a Python function lacks a return statement, Python actually generates - instructions to pop the result of the last statement off the stack, push - None onto the stack, and then return that. If we really don't care what - value is returned, then we can save two instructions by returning the - result of the last statement. It looks funny but it helps. -* We assume that type and bounds checking has happened at a higher level. -""" - -__author__ = 'kenton@google.com (Kenton Varda)' - -import struct -from google.protobuf.internal import wire_format - - -# This will overflow and thus become IEEE-754 "infinity". We would use -# "float('inf')" but it doesn't work on Windows pre-Python-2.6. -_POS_INF = 1e10000 -_NEG_INF = -_POS_INF - - -def _VarintSize(value): - """Compute the size of a varint value.""" - if value <= 0x7f: return 1 - if value <= 0x3fff: return 2 - if value <= 0x1fffff: return 3 - if value <= 0xfffffff: return 4 - if value <= 0x7ffffffff: return 5 - if value <= 0x3ffffffffff: return 6 - if value <= 0x1ffffffffffff: return 7 - if value <= 0xffffffffffffff: return 8 - if value <= 0x7fffffffffffffff: return 9 - return 10 - - -def _SignedVarintSize(value): - """Compute the size of a signed varint value.""" - if value < 0: return 10 - if value <= 0x7f: return 1 - if value <= 0x3fff: return 2 - if value <= 0x1fffff: return 3 - if value <= 0xfffffff: return 4 - if value <= 0x7ffffffff: return 5 - if value <= 0x3ffffffffff: return 6 - if value <= 0x1ffffffffffff: return 7 - if value <= 0xffffffffffffff: return 8 - if value <= 0x7fffffffffffffff: return 9 - return 10 - - -def _TagSize(field_number): - """Returns the number of bytes required to serialize a tag with this field - number.""" - # Just pass in type 0, since the type won't affect the tag+type size. - return _VarintSize(wire_format.PackTag(field_number, 0)) - - -# -------------------------------------------------------------------- -# In this section we define some generic sizers. Each of these functions -# takes parameters specific to a particular field type, e.g. int32 or fixed64. -# It returns another function which in turn takes parameters specific to a -# particular field, e.g. the field number and whether it is repeated or packed. -# Look at the next section to see how these are used. - - -def _SimpleSizer(compute_value_size): - """A sizer which uses the function compute_value_size to compute the size of - each value. Typically compute_value_size is _VarintSize.""" - - def SpecificSizer(field_number, is_repeated, is_packed): - tag_size = _TagSize(field_number) - if is_packed: - local_VarintSize = _VarintSize - def PackedFieldSize(value): - result = 0 - for element in value: - result += compute_value_size(element) - return result + local_VarintSize(result) + tag_size - return PackedFieldSize - elif is_repeated: - def RepeatedFieldSize(value): - result = tag_size * len(value) - for element in value: - result += compute_value_size(element) - return result - return RepeatedFieldSize - else: - def FieldSize(value): - return tag_size + compute_value_size(value) - return FieldSize - - return SpecificSizer - - -def _ModifiedSizer(compute_value_size, modify_value): - """Like SimpleSizer, but modify_value is invoked on each value before it is - passed to compute_value_size. modify_value is typically ZigZagEncode.""" - - def SpecificSizer(field_number, is_repeated, is_packed): - tag_size = _TagSize(field_number) - if is_packed: - local_VarintSize = _VarintSize - def PackedFieldSize(value): - result = 0 - for element in value: - result += compute_value_size(modify_value(element)) - return result + local_VarintSize(result) + tag_size - return PackedFieldSize - elif is_repeated: - def RepeatedFieldSize(value): - result = tag_size * len(value) - for element in value: - result += compute_value_size(modify_value(element)) - return result - return RepeatedFieldSize - else: - def FieldSize(value): - return tag_size + compute_value_size(modify_value(value)) - return FieldSize - - return SpecificSizer - - -def _FixedSizer(value_size): - """Like _SimpleSizer except for a fixed-size field. The input is the size - of one value.""" - - def SpecificSizer(field_number, is_repeated, is_packed): - tag_size = _TagSize(field_number) - if is_packed: - local_VarintSize = _VarintSize - def PackedFieldSize(value): - result = len(value) * value_size - return result + local_VarintSize(result) + tag_size - return PackedFieldSize - elif is_repeated: - element_size = value_size + tag_size - def RepeatedFieldSize(value): - return len(value) * element_size - return RepeatedFieldSize - else: - field_size = value_size + tag_size - def FieldSize(value): - return field_size - return FieldSize - - return SpecificSizer - - -# ==================================================================== -# Here we declare a sizer constructor for each field type. Each "sizer -# constructor" is a function that takes (field_number, is_repeated, is_packed) -# as parameters and returns a sizer, which in turn takes a field value as -# a parameter and returns its encoded size. - - -Int32Sizer = Int64Sizer = EnumSizer = _SimpleSizer(_SignedVarintSize) - -UInt32Sizer = UInt64Sizer = _SimpleSizer(_VarintSize) - -SInt32Sizer = SInt64Sizer = _ModifiedSizer( - _SignedVarintSize, wire_format.ZigZagEncode) - -Fixed32Sizer = SFixed32Sizer = FloatSizer = _FixedSizer(4) -Fixed64Sizer = SFixed64Sizer = DoubleSizer = _FixedSizer(8) - -BoolSizer = _FixedSizer(1) - - -def StringSizer(field_number, is_repeated, is_packed): - """Returns a sizer for a string field.""" - - tag_size = _TagSize(field_number) - local_VarintSize = _VarintSize - local_len = len - assert not is_packed - if is_repeated: - def RepeatedFieldSize(value): - result = tag_size * len(value) - for element in value: - l = local_len(element.encode('utf-8')) - result += local_VarintSize(l) + l - return result - return RepeatedFieldSize - else: - def FieldSize(value): - l = local_len(value.encode('utf-8')) - return tag_size + local_VarintSize(l) + l - return FieldSize - - -def BytesSizer(field_number, is_repeated, is_packed): - """Returns a sizer for a bytes field.""" - - tag_size = _TagSize(field_number) - local_VarintSize = _VarintSize - local_len = len - assert not is_packed - if is_repeated: - def RepeatedFieldSize(value): - result = tag_size * len(value) - for element in value: - l = local_len(element) - result += local_VarintSize(l) + l - return result - return RepeatedFieldSize - else: - def FieldSize(value): - l = local_len(value) - return tag_size + local_VarintSize(l) + l - return FieldSize - - -def GroupSizer(field_number, is_repeated, is_packed): - """Returns a sizer for a group field.""" - - tag_size = _TagSize(field_number) * 2 - assert not is_packed - if is_repeated: - def RepeatedFieldSize(value): - result = tag_size * len(value) - for element in value: - result += element.ByteSize() - return result - return RepeatedFieldSize - else: - def FieldSize(value): - return tag_size + value.ByteSize() - return FieldSize - - -def MessageSizer(field_number, is_repeated, is_packed): - """Returns a sizer for a message field.""" - - tag_size = _TagSize(field_number) - local_VarintSize = _VarintSize - assert not is_packed - if is_repeated: - def RepeatedFieldSize(value): - result = tag_size * len(value) - for element in value: - l = element.ByteSize() - result += local_VarintSize(l) + l - return result - return RepeatedFieldSize - else: - def FieldSize(value): - l = value.ByteSize() - return tag_size + local_VarintSize(l) + l - return FieldSize - - -# -------------------------------------------------------------------- -# MessageSet is special. - - -def MessageSetItemSizer(field_number): - """Returns a sizer for extensions of MessageSet. - - The message set message looks like this: - message MessageSet { - repeated group Item = 1 { - required int32 type_id = 2; - required string message = 3; - } - } - """ - static_size = (_TagSize(1) * 2 + _TagSize(2) + _VarintSize(field_number) + - _TagSize(3)) - local_VarintSize = _VarintSize - - def FieldSize(value): - l = value.ByteSize() - return static_size + local_VarintSize(l) + l - - return FieldSize - - -# ==================================================================== -# Encoders! - - -def _VarintEncoder(): - """Return an encoder for a basic varint value (does not include tag).""" - - local_chr = chr - def EncodeVarint(write, value): - bits = value & 0x7f - value >>= 7 - while value: - write(local_chr(0x80|bits)) - bits = value & 0x7f - value >>= 7 - return write(local_chr(bits)) - - return EncodeVarint - - -def _SignedVarintEncoder(): - """Return an encoder for a basic signed varint value (does not include - tag).""" - - local_chr = chr - def EncodeSignedVarint(write, value): - if value < 0: - value += (1 << 64) - bits = value & 0x7f - value >>= 7 - while value: - write(local_chr(0x80|bits)) - bits = value & 0x7f - value >>= 7 - return write(local_chr(bits)) - - return EncodeSignedVarint - - -_EncodeVarint = _VarintEncoder() -_EncodeSignedVarint = _SignedVarintEncoder() - - -def _VarintBytes(value): - """Encode the given integer as a varint and return the bytes. This is only - called at startup time so it doesn't need to be fast.""" - - pieces = [] - _EncodeVarint(pieces.append, value) - return "".join(pieces) - - -def TagBytes(field_number, wire_type): - """Encode the given tag and return the bytes. Only called at startup.""" - - return _VarintBytes(wire_format.PackTag(field_number, wire_type)) - -# -------------------------------------------------------------------- -# As with sizers (see above), we have a number of common encoder -# implementations. - - -def _SimpleEncoder(wire_type, encode_value, compute_value_size): - """Return a constructor for an encoder for fields of a particular type. - - Args: - wire_type: The field's wire type, for encoding tags. - encode_value: A function which encodes an individual value, e.g. - _EncodeVarint(). - compute_value_size: A function which computes the size of an individual - value, e.g. _VarintSize(). - """ - - def SpecificEncoder(field_number, is_repeated, is_packed): - if is_packed: - tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) - local_EncodeVarint = _EncodeVarint - def EncodePackedField(write, value): - write(tag_bytes) - size = 0 - for element in value: - size += compute_value_size(element) - local_EncodeVarint(write, size) - for element in value: - encode_value(write, element) - return EncodePackedField - elif is_repeated: - tag_bytes = TagBytes(field_number, wire_type) - def EncodeRepeatedField(write, value): - for element in value: - write(tag_bytes) - encode_value(write, element) - return EncodeRepeatedField - else: - tag_bytes = TagBytes(field_number, wire_type) - def EncodeField(write, value): - write(tag_bytes) - return encode_value(write, value) - return EncodeField - - return SpecificEncoder - - -def _ModifiedEncoder(wire_type, encode_value, compute_value_size, modify_value): - """Like SimpleEncoder but additionally invokes modify_value on every value - before passing it to encode_value. Usually modify_value is ZigZagEncode.""" - - def SpecificEncoder(field_number, is_repeated, is_packed): - if is_packed: - tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) - local_EncodeVarint = _EncodeVarint - def EncodePackedField(write, value): - write(tag_bytes) - size = 0 - for element in value: - size += compute_value_size(modify_value(element)) - local_EncodeVarint(write, size) - for element in value: - encode_value(write, modify_value(element)) - return EncodePackedField - elif is_repeated: - tag_bytes = TagBytes(field_number, wire_type) - def EncodeRepeatedField(write, value): - for element in value: - write(tag_bytes) - encode_value(write, modify_value(element)) - return EncodeRepeatedField - else: - tag_bytes = TagBytes(field_number, wire_type) - def EncodeField(write, value): - write(tag_bytes) - return encode_value(write, modify_value(value)) - return EncodeField - - return SpecificEncoder - - -def _StructPackEncoder(wire_type, format): - """Return a constructor for an encoder for a fixed-width field. - - Args: - wire_type: The field's wire type, for encoding tags. - format: The format string to pass to struct.pack(). - """ - - value_size = struct.calcsize(format) - - def SpecificEncoder(field_number, is_repeated, is_packed): - local_struct_pack = struct.pack - if is_packed: - tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) - local_EncodeVarint = _EncodeVarint - def EncodePackedField(write, value): - write(tag_bytes) - local_EncodeVarint(write, len(value) * value_size) - for element in value: - write(local_struct_pack(format, element)) - return EncodePackedField - elif is_repeated: - tag_bytes = TagBytes(field_number, wire_type) - def EncodeRepeatedField(write, value): - for element in value: - write(tag_bytes) - write(local_struct_pack(format, element)) - return EncodeRepeatedField - else: - tag_bytes = TagBytes(field_number, wire_type) - def EncodeField(write, value): - write(tag_bytes) - return write(local_struct_pack(format, value)) - return EncodeField - - return SpecificEncoder - - -def _FloatingPointEncoder(wire_type, format): - """Return a constructor for an encoder for float fields. - - This is like StructPackEncoder, but catches errors that may be due to - passing non-finite floating-point values to struct.pack, and makes a - second attempt to encode those values. - - Args: - wire_type: The field's wire type, for encoding tags. - format: The format string to pass to struct.pack(). - """ - - value_size = struct.calcsize(format) - if value_size == 4: - def EncodeNonFiniteOrRaise(write, value): - # Remember that the serialized form uses little-endian byte order. - if value == _POS_INF: - write('\x00\x00\x80\x7F') - elif value == _NEG_INF: - write('\x00\x00\x80\xFF') - elif value != value: # NaN - write('\x00\x00\xC0\x7F') - else: - raise - elif value_size == 8: - def EncodeNonFiniteOrRaise(write, value): - if value == _POS_INF: - write('\x00\x00\x00\x00\x00\x00\xF0\x7F') - elif value == _NEG_INF: - write('\x00\x00\x00\x00\x00\x00\xF0\xFF') - elif value != value: # NaN - write('\x00\x00\x00\x00\x00\x00\xF8\x7F') - else: - raise - else: - raise ValueError('Can\'t encode floating-point values that are ' - '%d bytes long (only 4 or 8)' % value_size) - - def SpecificEncoder(field_number, is_repeated, is_packed): - local_struct_pack = struct.pack - if is_packed: - tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) - local_EncodeVarint = _EncodeVarint - def EncodePackedField(write, value): - write(tag_bytes) - local_EncodeVarint(write, len(value) * value_size) - for element in value: - # This try/except block is going to be faster than any code that - # we could write to check whether element is finite. - try: - write(local_struct_pack(format, element)) - except SystemError: - EncodeNonFiniteOrRaise(write, element) - return EncodePackedField - elif is_repeated: - tag_bytes = TagBytes(field_number, wire_type) - def EncodeRepeatedField(write, value): - for element in value: - write(tag_bytes) - try: - write(local_struct_pack(format, element)) - except SystemError: - EncodeNonFiniteOrRaise(write, element) - return EncodeRepeatedField - else: - tag_bytes = TagBytes(field_number, wire_type) - def EncodeField(write, value): - write(tag_bytes) - try: - write(local_struct_pack(format, value)) - except SystemError: - EncodeNonFiniteOrRaise(write, value) - return EncodeField - - return SpecificEncoder - - -# ==================================================================== -# Here we declare an encoder constructor for each field type. These work -# very similarly to sizer constructors, described earlier. - - -Int32Encoder = Int64Encoder = EnumEncoder = _SimpleEncoder( - wire_format.WIRETYPE_VARINT, _EncodeSignedVarint, _SignedVarintSize) - -UInt32Encoder = UInt64Encoder = _SimpleEncoder( - wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize) - -SInt32Encoder = SInt64Encoder = _ModifiedEncoder( - wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize, - wire_format.ZigZagEncode) - -# Note that Python conveniently guarantees that when using the '<' prefix on -# formats, they will also have the same size across all platforms (as opposed -# to without the prefix, where their sizes depend on the C compiler's basic -# type sizes). -Fixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<I') -Fixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<Q') -SFixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, '<i') -SFixed64Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED64, '<q') -FloatEncoder = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED32, '<f') -DoubleEncoder = _FloatingPointEncoder(wire_format.WIRETYPE_FIXED64, '<d') - - -def BoolEncoder(field_number, is_repeated, is_packed): - """Returns an encoder for a boolean field.""" - - false_byte = chr(0) - true_byte = chr(1) - if is_packed: - tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) - local_EncodeVarint = _EncodeVarint - def EncodePackedField(write, value): - write(tag_bytes) - local_EncodeVarint(write, len(value)) - for element in value: - if element: - write(true_byte) - else: - write(false_byte) - return EncodePackedField - elif is_repeated: - tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT) - def EncodeRepeatedField(write, value): - for element in value: - write(tag_bytes) - if element: - write(true_byte) - else: - write(false_byte) - return EncodeRepeatedField - else: - tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_VARINT) - def EncodeField(write, value): - write(tag_bytes) - if value: - return write(true_byte) - return write(false_byte) - return EncodeField - - -def StringEncoder(field_number, is_repeated, is_packed): - """Returns an encoder for a string field.""" - - tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) - local_EncodeVarint = _EncodeVarint - local_len = len - assert not is_packed - if is_repeated: - def EncodeRepeatedField(write, value): - for element in value: - encoded = element.encode('utf-8') - write(tag) - local_EncodeVarint(write, local_len(encoded)) - write(encoded) - return EncodeRepeatedField - else: - def EncodeField(write, value): - encoded = value.encode('utf-8') - write(tag) - local_EncodeVarint(write, local_len(encoded)) - return write(encoded) - return EncodeField - - -def BytesEncoder(field_number, is_repeated, is_packed): - """Returns an encoder for a bytes field.""" - - tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) - local_EncodeVarint = _EncodeVarint - local_len = len - assert not is_packed - if is_repeated: - def EncodeRepeatedField(write, value): - for element in value: - write(tag) - local_EncodeVarint(write, local_len(element)) - write(element) - return EncodeRepeatedField - else: - def EncodeField(write, value): - write(tag) - local_EncodeVarint(write, local_len(value)) - return write(value) - return EncodeField - - -def GroupEncoder(field_number, is_repeated, is_packed): - """Returns an encoder for a group field.""" - - start_tag = TagBytes(field_number, wire_format.WIRETYPE_START_GROUP) - end_tag = TagBytes(field_number, wire_format.WIRETYPE_END_GROUP) - assert not is_packed - if is_repeated: - def EncodeRepeatedField(write, value): - for element in value: - write(start_tag) - element._InternalSerialize(write) - write(end_tag) - return EncodeRepeatedField - else: - def EncodeField(write, value): - write(start_tag) - value._InternalSerialize(write) - return write(end_tag) - return EncodeField - - -def MessageEncoder(field_number, is_repeated, is_packed): - """Returns an encoder for a message field.""" - - tag = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) - local_EncodeVarint = _EncodeVarint - assert not is_packed - if is_repeated: - def EncodeRepeatedField(write, value): - for element in value: - write(tag) - local_EncodeVarint(write, element.ByteSize()) - element._InternalSerialize(write) - return EncodeRepeatedField - else: - def EncodeField(write, value): - write(tag) - local_EncodeVarint(write, value.ByteSize()) - return value._InternalSerialize(write) - return EncodeField - - -# -------------------------------------------------------------------- -# As before, MessageSet is special. - - -def MessageSetItemEncoder(field_number): - """Encoder for extensions of MessageSet. - - The message set message looks like this: - message MessageSet { - repeated group Item = 1 { - required int32 type_id = 2; - required string message = 3; - } - } - """ - start_bytes = "".join([ - TagBytes(1, wire_format.WIRETYPE_START_GROUP), - TagBytes(2, wire_format.WIRETYPE_VARINT), - _VarintBytes(field_number), - TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED)]) - end_bytes = TagBytes(1, wire_format.WIRETYPE_END_GROUP) - local_EncodeVarint = _EncodeVarint - - def EncodeField(write, value): - write(start_bytes) - local_EncodeVarint(write, value.ByteSize()) - value._InternalSerialize(write) - return write(end_bytes) - - return EncodeField diff --git a/generator/google/protobuf/internal/enum_type_wrapper.py b/generator/google/protobuf/internal/enum_type_wrapper.py deleted file mode 100644 index 7b28645..0000000 --- a/generator/google/protobuf/internal/enum_type_wrapper.py +++ /dev/null @@ -1,89 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""A simple wrapper around enum types to expose utility functions. - -Instances are created as properties with the same name as the enum they wrap -on proto classes. For usage, see: - reflection_test.py -""" - -__author__ = 'rabsatt@google.com (Kevin Rabsatt)' - - -class EnumTypeWrapper(object): - """A utility for finding the names of enum values.""" - - DESCRIPTOR = None - - def __init__(self, enum_type): - """Inits EnumTypeWrapper with an EnumDescriptor.""" - self._enum_type = enum_type - self.DESCRIPTOR = enum_type; - - def Name(self, number): - """Returns a string containing the name of an enum value.""" - if number in self._enum_type.values_by_number: - return self._enum_type.values_by_number[number].name - raise ValueError('Enum %s has no name defined for value %d' % ( - self._enum_type.name, number)) - - def Value(self, name): - """Returns the value coresponding to the given enum name.""" - if name in self._enum_type.values_by_name: - return self._enum_type.values_by_name[name].number - raise ValueError('Enum %s has no value defined for name %s' % ( - self._enum_type.name, name)) - - def keys(self): - """Return a list of the string names in the enum. - - These are returned in the order they were defined in the .proto file. - """ - - return [value_descriptor.name - for value_descriptor in self._enum_type.values] - - def values(self): - """Return a list of the integer values in the enum. - - These are returned in the order they were defined in the .proto file. - """ - - return [value_descriptor.number - for value_descriptor in self._enum_type.values] - - def items(self): - """Return a list of the (name, value) pairs of the enum. - - These are returned in the order they were defined in the .proto file. - """ - return [(value_descriptor.name, value_descriptor.number) - for value_descriptor in self._enum_type.values] diff --git a/generator/google/protobuf/internal/message_listener.py b/generator/google/protobuf/internal/message_listener.py deleted file mode 100644 index 1080234..0000000 --- a/generator/google/protobuf/internal/message_listener.py +++ /dev/null @@ -1,78 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Defines a listener interface for observing certain -state transitions on Message objects. - -Also defines a null implementation of this interface. -""" - -__author__ = 'robinson@google.com (Will Robinson)' - - -class MessageListener(object): - - """Listens for modifications made to a message. Meant to be registered via - Message._SetListener(). - - Attributes: - dirty: If True, then calling Modified() would be a no-op. This can be - used to avoid these calls entirely in the common case. - """ - - def Modified(self): - """Called every time the message is modified in such a way that the parent - message may need to be updated. This currently means either: - (a) The message was modified for the first time, so the parent message - should henceforth mark the message as present. - (b) The message's cached byte size became dirty -- i.e. the message was - modified for the first time after a previous call to ByteSize(). - Therefore the parent should also mark its byte size as dirty. - Note that (a) implies (b), since new objects start out with a client cached - size (zero). However, we document (a) explicitly because it is important. - - Modified() will *only* be called in response to one of these two events -- - not every time the sub-message is modified. - - Note that if the listener's |dirty| attribute is true, then calling - Modified at the moment would be a no-op, so it can be skipped. Performance- - sensitive callers should check this attribute directly before calling since - it will be true most of the time. - """ - - raise NotImplementedError - - -class NullMessageListener(object): - - """No-op MessageListener implementation.""" - - def Modified(self): - pass diff --git a/generator/google/protobuf/internal/python_message.py b/generator/google/protobuf/internal/python_message.py deleted file mode 100644 index 4bea57a..0000000 --- a/generator/google/protobuf/internal/python_message.py +++ /dev/null @@ -1,1150 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# This code is meant to work on Python 2.4 and above only. -# -# TODO(robinson): Helpers for verbose, common checks like seeing if a -# descriptor's cpp_type is CPPTYPE_MESSAGE. - -"""Contains a metaclass and helper functions used to create -protocol message classes from Descriptor objects at runtime. - -Recall that a metaclass is the "type" of a class. -(A class is to a metaclass what an instance is to a class.) - -In this case, we use the GeneratedProtocolMessageType metaclass -to inject all the useful functionality into the classes -output by the protocol compiler at compile-time. - -The upshot of all this is that the real implementation -details for ALL pure-Python protocol buffers are *here in -this file*. -""" - -__author__ = 'robinson@google.com (Will Robinson)' - -try: - from cStringIO import StringIO -except ImportError: - from StringIO import StringIO -import copy_reg -import struct -import weakref - -# We use "as" to avoid name collisions with variables. -from google.protobuf.internal import containers -from google.protobuf.internal import decoder -from google.protobuf.internal import encoder -from google.protobuf.internal import enum_type_wrapper -from google.protobuf.internal import message_listener as message_listener_mod -from google.protobuf.internal import type_checkers -from google.protobuf.internal import wire_format -from google.protobuf import descriptor as descriptor_mod -from google.protobuf import message as message_mod -from google.protobuf import text_format - -_FieldDescriptor = descriptor_mod.FieldDescriptor - - -def NewMessage(bases, descriptor, dictionary): - _AddClassAttributesForNestedExtensions(descriptor, dictionary) - _AddSlots(descriptor, dictionary) - return bases - - -def InitMessage(descriptor, cls): - cls._decoders_by_tag = {} - cls._extensions_by_name = {} - cls._extensions_by_number = {} - if (descriptor.has_options and - descriptor.GetOptions().message_set_wire_format): - cls._decoders_by_tag[decoder.MESSAGE_SET_ITEM_TAG] = ( - decoder.MessageSetItemDecoder(cls._extensions_by_number)) - - # Attach stuff to each FieldDescriptor for quick lookup later on. - for field in descriptor.fields: - _AttachFieldHelpers(cls, field) - - _AddEnumValues(descriptor, cls) - _AddInitMethod(descriptor, cls) - _AddPropertiesForFields(descriptor, cls) - _AddPropertiesForExtensions(descriptor, cls) - _AddStaticMethods(cls) - _AddMessageMethods(descriptor, cls) - _AddPrivateHelperMethods(cls) - copy_reg.pickle(cls, lambda obj: (cls, (), obj.__getstate__())) - - -# Stateless helpers for GeneratedProtocolMessageType below. -# Outside clients should not access these directly. -# -# I opted not to make any of these methods on the metaclass, to make it more -# clear that I'm not really using any state there and to keep clients from -# thinking that they have direct access to these construction helpers. - - -def _PropertyName(proto_field_name): - """Returns the name of the public property attribute which - clients can use to get and (in some cases) set the value - of a protocol message field. - - Args: - proto_field_name: The protocol message field name, exactly - as it appears (or would appear) in a .proto file. - """ - # TODO(robinson): Escape Python keywords (e.g., yield), and test this support. - # nnorwitz makes my day by writing: - # """ - # FYI. See the keyword module in the stdlib. This could be as simple as: - # - # if keyword.iskeyword(proto_field_name): - # return proto_field_name + "_" - # return proto_field_name - # """ - # Kenton says: The above is a BAD IDEA. People rely on being able to use - # getattr() and setattr() to reflectively manipulate field values. If we - # rename the properties, then every such user has to also make sure to apply - # the same transformation. Note that currently if you name a field "yield", - # you can still access it just fine using getattr/setattr -- it's not even - # that cumbersome to do so. - # TODO(kenton): Remove this method entirely if/when everyone agrees with my - # position. - return proto_field_name - - -def _VerifyExtensionHandle(message, extension_handle): - """Verify that the given extension handle is valid.""" - - if not isinstance(extension_handle, _FieldDescriptor): - raise KeyError('HasExtension() expects an extension handle, got: %s' % - extension_handle) - - if not extension_handle.is_extension: - raise KeyError('"%s" is not an extension.' % extension_handle.full_name) - - if not extension_handle.containing_type: - raise KeyError('"%s" is missing a containing_type.' - % extension_handle.full_name) - - if extension_handle.containing_type is not message.DESCRIPTOR: - raise KeyError('Extension "%s" extends message type "%s", but this ' - 'message is of type "%s".' % - (extension_handle.full_name, - extension_handle.containing_type.full_name, - message.DESCRIPTOR.full_name)) - - -def _AddSlots(message_descriptor, dictionary): - """Adds a __slots__ entry to dictionary, containing the names of all valid - attributes for this message type. - - Args: - message_descriptor: A Descriptor instance describing this message type. - dictionary: Class dictionary to which we'll add a '__slots__' entry. - """ - dictionary['__slots__'] = ['_cached_byte_size', - '_cached_byte_size_dirty', - '_fields', - '_unknown_fields', - '_is_present_in_parent', - '_listener', - '_listener_for_children', - '__weakref__'] - - -def _IsMessageSetExtension(field): - return (field.is_extension and - field.containing_type.has_options and - field.containing_type.GetOptions().message_set_wire_format and - field.type == _FieldDescriptor.TYPE_MESSAGE and - field.message_type == field.extension_scope and - field.label == _FieldDescriptor.LABEL_OPTIONAL) - - -def _AttachFieldHelpers(cls, field_descriptor): - is_repeated = (field_descriptor.label == _FieldDescriptor.LABEL_REPEATED) - is_packed = (field_descriptor.has_options and - field_descriptor.GetOptions().packed) - - if _IsMessageSetExtension(field_descriptor): - field_encoder = encoder.MessageSetItemEncoder(field_descriptor.number) - sizer = encoder.MessageSetItemSizer(field_descriptor.number) - else: - field_encoder = type_checkers.TYPE_TO_ENCODER[field_descriptor.type]( - field_descriptor.number, is_repeated, is_packed) - sizer = type_checkers.TYPE_TO_SIZER[field_descriptor.type]( - field_descriptor.number, is_repeated, is_packed) - - field_descriptor._encoder = field_encoder - field_descriptor._sizer = sizer - field_descriptor._default_constructor = _DefaultValueConstructorForField( - field_descriptor) - - def AddDecoder(wiretype, is_packed): - tag_bytes = encoder.TagBytes(field_descriptor.number, wiretype) - cls._decoders_by_tag[tag_bytes] = ( - type_checkers.TYPE_TO_DECODER[field_descriptor.type]( - field_descriptor.number, is_repeated, is_packed, - field_descriptor, field_descriptor._default_constructor)) - - AddDecoder(type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type], - False) - - if is_repeated and wire_format.IsTypePackable(field_descriptor.type): - # To support wire compatibility of adding packed = true, add a decoder for - # packed values regardless of the field's options. - AddDecoder(wire_format.WIRETYPE_LENGTH_DELIMITED, True) - - -def _AddClassAttributesForNestedExtensions(descriptor, dictionary): - extension_dict = descriptor.extensions_by_name - for extension_name, extension_field in extension_dict.iteritems(): - assert extension_name not in dictionary - dictionary[extension_name] = extension_field - - -def _AddEnumValues(descriptor, cls): - """Sets class-level attributes for all enum fields defined in this message. - - Also exporting a class-level object that can name enum values. - - Args: - descriptor: Descriptor object for this message type. - cls: Class we're constructing for this message type. - """ - for enum_type in descriptor.enum_types: - setattr(cls, enum_type.name, enum_type_wrapper.EnumTypeWrapper(enum_type)) - for enum_value in enum_type.values: - setattr(cls, enum_value.name, enum_value.number) - - -def _DefaultValueConstructorForField(field): - """Returns a function which returns a default value for a field. - - Args: - field: FieldDescriptor object for this field. - - The returned function has one argument: - message: Message instance containing this field, or a weakref proxy - of same. - - That function in turn returns a default value for this field. The default - value may refer back to |message| via a weak reference. - """ - - if field.label == _FieldDescriptor.LABEL_REPEATED: - if field.has_default_value and field.default_value != []: - raise ValueError('Repeated field default value not empty list: %s' % ( - field.default_value)) - if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: - # We can't look at _concrete_class yet since it might not have - # been set. (Depends on order in which we initialize the classes). - message_type = field.message_type - def MakeRepeatedMessageDefault(message): - return containers.RepeatedCompositeFieldContainer( - message._listener_for_children, field.message_type) - return MakeRepeatedMessageDefault - else: - type_checker = type_checkers.GetTypeChecker(field.cpp_type, field.type) - def MakeRepeatedScalarDefault(message): - return containers.RepeatedScalarFieldContainer( - message._listener_for_children, type_checker) - return MakeRepeatedScalarDefault - - if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: - # _concrete_class may not yet be initialized. - message_type = field.message_type - def MakeSubMessageDefault(message): - result = message_type._concrete_class() - result._SetListener(message._listener_for_children) - return result - return MakeSubMessageDefault - - def MakeScalarDefault(message): - # TODO(protobuf-team): This may be broken since there may not be - # default_value. Combine with has_default_value somehow. - return field.default_value - return MakeScalarDefault - - -def _AddInitMethod(message_descriptor, cls): - """Adds an __init__ method to cls.""" - fields = message_descriptor.fields - def init(self, **kwargs): - self._cached_byte_size = 0 - self._cached_byte_size_dirty = len(kwargs) > 0 - self._fields = {} - # _unknown_fields is () when empty for efficiency, and will be turned into - # a list if fields are added. - self._unknown_fields = () - self._is_present_in_parent = False - self._listener = message_listener_mod.NullMessageListener() - self._listener_for_children = _Listener(self) - for field_name, field_value in kwargs.iteritems(): - field = _GetFieldByName(message_descriptor, field_name) - if field is None: - raise TypeError("%s() got an unexpected keyword argument '%s'" % - (message_descriptor.name, field_name)) - if field.label == _FieldDescriptor.LABEL_REPEATED: - copy = field._default_constructor(self) - if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: # Composite - for val in field_value: - copy.add().MergeFrom(val) - else: # Scalar - copy.extend(field_value) - self._fields[field] = copy - elif field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: - copy = field._default_constructor(self) - copy.MergeFrom(field_value) - self._fields[field] = copy - else: - setattr(self, field_name, field_value) - - init.__module__ = None - init.__doc__ = None - cls.__init__ = init - - -def _GetFieldByName(message_descriptor, field_name): - """Returns a field descriptor by field name. - - Args: - message_descriptor: A Descriptor describing all fields in message. - field_name: The name of the field to retrieve. - Returns: - The field descriptor associated with the field name. - """ - try: - return message_descriptor.fields_by_name[field_name] - except KeyError: - raise ValueError('Protocol message has no "%s" field.' % field_name) - - -def _AddPropertiesForFields(descriptor, cls): - """Adds properties for all fields in this protocol message type.""" - for field in descriptor.fields: - _AddPropertiesForField(field, cls) - - if descriptor.is_extendable: - # _ExtensionDict is just an adaptor with no state so we allocate a new one - # every time it is accessed. - cls.Extensions = property(lambda self: _ExtensionDict(self)) - - -def _AddPropertiesForField(field, cls): - """Adds a public property for a protocol message field. - Clients can use this property to get and (in the case - of non-repeated scalar fields) directly set the value - of a protocol message field. - - Args: - field: A FieldDescriptor for this field. - cls: The class we're constructing. - """ - # Catch it if we add other types that we should - # handle specially here. - assert _FieldDescriptor.MAX_CPPTYPE == 10 - - constant_name = field.name.upper() + "_FIELD_NUMBER" - setattr(cls, constant_name, field.number) - - if field.label == _FieldDescriptor.LABEL_REPEATED: - _AddPropertiesForRepeatedField(field, cls) - elif field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: - _AddPropertiesForNonRepeatedCompositeField(field, cls) - else: - _AddPropertiesForNonRepeatedScalarField(field, cls) - - -def _AddPropertiesForRepeatedField(field, cls): - """Adds a public property for a "repeated" protocol message field. Clients - can use this property to get the value of the field, which will be either a - _RepeatedScalarFieldContainer or _RepeatedCompositeFieldContainer (see - below). - - Note that when clients add values to these containers, we perform - type-checking in the case of repeated scalar fields, and we also set any - necessary "has" bits as a side-effect. - - Args: - field: A FieldDescriptor for this field. - cls: The class we're constructing. - """ - proto_field_name = field.name - property_name = _PropertyName(proto_field_name) - - def getter(self): - field_value = self._fields.get(field) - if field_value is None: - # Construct a new object to represent this field. - field_value = field._default_constructor(self) - - # Atomically check if another thread has preempted us and, if not, swap - # in the new object we just created. If someone has preempted us, we - # take that object and discard ours. - # WARNING: We are relying on setdefault() being atomic. This is true - # in CPython but we haven't investigated others. This warning appears - # in several other locations in this file. - field_value = self._fields.setdefault(field, field_value) - return field_value - getter.__module__ = None - getter.__doc__ = 'Getter for %s.' % proto_field_name - - # We define a setter just so we can throw an exception with a more - # helpful error message. - def setter(self, new_value): - raise AttributeError('Assignment not allowed to repeated field ' - '"%s" in protocol message object.' % proto_field_name) - - doc = 'Magic attribute generated for "%s" proto field.' % proto_field_name - setattr(cls, property_name, property(getter, setter, doc=doc)) - - -def _AddPropertiesForNonRepeatedScalarField(field, cls): - """Adds a public property for a nonrepeated, scalar protocol message field. - Clients can use this property to get and directly set the value of the field. - Note that when the client sets the value of a field by using this property, - all necessary "has" bits are set as a side-effect, and we also perform - type-checking. - - Args: - field: A FieldDescriptor for this field. - cls: The class we're constructing. - """ - proto_field_name = field.name - property_name = _PropertyName(proto_field_name) - type_checker = type_checkers.GetTypeChecker(field.cpp_type, field.type) - default_value = field.default_value - valid_values = set() - - def getter(self): - # TODO(protobuf-team): This may be broken since there may not be - # default_value. Combine with has_default_value somehow. - return self._fields.get(field, default_value) - getter.__module__ = None - getter.__doc__ = 'Getter for %s.' % proto_field_name - def setter(self, new_value): - type_checker.CheckValue(new_value) - self._fields[field] = new_value - # Check _cached_byte_size_dirty inline to improve performance, since scalar - # setters are called frequently. - if not self._cached_byte_size_dirty: - self._Modified() - - setter.__module__ = None - setter.__doc__ = 'Setter for %s.' % proto_field_name - - # Add a property to encapsulate the getter/setter. - doc = 'Magic attribute generated for "%s" proto field.' % proto_field_name - setattr(cls, property_name, property(getter, setter, doc=doc)) - - -def _AddPropertiesForNonRepeatedCompositeField(field, cls): - """Adds a public property for a nonrepeated, composite protocol message field. - A composite field is a "group" or "message" field. - - Clients can use this property to get the value of the field, but cannot - assign to the property directly. - - Args: - field: A FieldDescriptor for this field. - cls: The class we're constructing. - """ - # TODO(robinson): Remove duplication with similar method - # for non-repeated scalars. - proto_field_name = field.name - property_name = _PropertyName(proto_field_name) - - # TODO(komarek): Can anyone explain to me why we cache the message_type this - # way, instead of referring to field.message_type inside of getter(self)? - # What if someone sets message_type later on (which makes for simpler - # dyanmic proto descriptor and class creation code). - message_type = field.message_type - - def getter(self): - field_value = self._fields.get(field) - if field_value is None: - # Construct a new object to represent this field. - field_value = message_type._concrete_class() # use field.message_type? - field_value._SetListener(self._listener_for_children) - - # Atomically check if another thread has preempted us and, if not, swap - # in the new object we just created. If someone has preempted us, we - # take that object and discard ours. - # WARNING: We are relying on setdefault() being atomic. This is true - # in CPython but we haven't investigated others. This warning appears - # in several other locations in this file. - field_value = self._fields.setdefault(field, field_value) - return field_value - getter.__module__ = None - getter.__doc__ = 'Getter for %s.' % proto_field_name - - # We define a setter just so we can throw an exception with a more - # helpful error message. - def setter(self, new_value): - raise AttributeError('Assignment not allowed to composite field ' - '"%s" in protocol message object.' % proto_field_name) - - # Add a property to encapsulate the getter. - doc = 'Magic attribute generated for "%s" proto field.' % proto_field_name - setattr(cls, property_name, property(getter, setter, doc=doc)) - - -def _AddPropertiesForExtensions(descriptor, cls): - """Adds properties for all fields in this protocol message type.""" - extension_dict = descriptor.extensions_by_name - for extension_name, extension_field in extension_dict.iteritems(): - constant_name = extension_name.upper() + "_FIELD_NUMBER" - setattr(cls, constant_name, extension_field.number) - - -def _AddStaticMethods(cls): - # TODO(robinson): This probably needs to be thread-safe(?) - def RegisterExtension(extension_handle): - extension_handle.containing_type = cls.DESCRIPTOR - _AttachFieldHelpers(cls, extension_handle) - - # Try to insert our extension, failing if an extension with the same number - # already exists. - actual_handle = cls._extensions_by_number.setdefault( - extension_handle.number, extension_handle) - if actual_handle is not extension_handle: - raise AssertionError( - 'Extensions "%s" and "%s" both try to extend message type "%s" with ' - 'field number %d.' % - (extension_handle.full_name, actual_handle.full_name, - cls.DESCRIPTOR.full_name, extension_handle.number)) - - cls._extensions_by_name[extension_handle.full_name] = extension_handle - - handle = extension_handle # avoid line wrapping - if _IsMessageSetExtension(handle): - # MessageSet extension. Also register under type name. - cls._extensions_by_name[ - extension_handle.message_type.full_name] = extension_handle - - cls.RegisterExtension = staticmethod(RegisterExtension) - - def FromString(s): - message = cls() - message.MergeFromString(s) - return message - cls.FromString = staticmethod(FromString) - - -def _IsPresent(item): - """Given a (FieldDescriptor, value) tuple from _fields, return true if the - value should be included in the list returned by ListFields().""" - - if item[0].label == _FieldDescriptor.LABEL_REPEATED: - return bool(item[1]) - elif item[0].cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: - return item[1]._is_present_in_parent - else: - return True - - -def _AddListFieldsMethod(message_descriptor, cls): - """Helper for _AddMessageMethods().""" - - def ListFields(self): - all_fields = [item for item in self._fields.iteritems() if _IsPresent(item)] - all_fields.sort(key = lambda item: item[0].number) - return all_fields - - cls.ListFields = ListFields - - -def _AddHasFieldMethod(message_descriptor, cls): - """Helper for _AddMessageMethods().""" - - singular_fields = {} - for field in message_descriptor.fields: - if field.label != _FieldDescriptor.LABEL_REPEATED: - singular_fields[field.name] = field - - def HasField(self, field_name): - try: - field = singular_fields[field_name] - except KeyError: - raise ValueError( - 'Protocol message has no singular "%s" field.' % field_name) - - if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: - value = self._fields.get(field) - return value is not None and value._is_present_in_parent - else: - return field in self._fields - cls.HasField = HasField - - -def _AddClearFieldMethod(message_descriptor, cls): - """Helper for _AddMessageMethods().""" - def ClearField(self, field_name): - try: - field = message_descriptor.fields_by_name[field_name] - except KeyError: - raise ValueError('Protocol message has no "%s" field.' % field_name) - - if field in self._fields: - # Note: If the field is a sub-message, its listener will still point - # at us. That's fine, because the worst than can happen is that it - # will call _Modified() and invalidate our byte size. Big deal. - del self._fields[field] - - # Always call _Modified() -- even if nothing was changed, this is - # a mutating method, and thus calling it should cause the field to become - # present in the parent message. - self._Modified() - - cls.ClearField = ClearField - - -def _AddClearExtensionMethod(cls): - """Helper for _AddMessageMethods().""" - def ClearExtension(self, extension_handle): - _VerifyExtensionHandle(self, extension_handle) - - # Similar to ClearField(), above. - if extension_handle in self._fields: - del self._fields[extension_handle] - self._Modified() - cls.ClearExtension = ClearExtension - - -def _AddClearMethod(message_descriptor, cls): - """Helper for _AddMessageMethods().""" - def Clear(self): - # Clear fields. - self._fields = {} - self._unknown_fields = () - self._Modified() - cls.Clear = Clear - - -def _AddHasExtensionMethod(cls): - """Helper for _AddMessageMethods().""" - def HasExtension(self, extension_handle): - _VerifyExtensionHandle(self, extension_handle) - if extension_handle.label == _FieldDescriptor.LABEL_REPEATED: - raise KeyError('"%s" is repeated.' % extension_handle.full_name) - - if extension_handle.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: - value = self._fields.get(extension_handle) - return value is not None and value._is_present_in_parent - else: - return extension_handle in self._fields - cls.HasExtension = HasExtension - - -def _AddEqualsMethod(message_descriptor, cls): - """Helper for _AddMessageMethods().""" - def __eq__(self, other): - if (not isinstance(other, message_mod.Message) or - other.DESCRIPTOR != self.DESCRIPTOR): - return False - - if self is other: - return True - - if not self.ListFields() == other.ListFields(): - return False - - # Sort unknown fields because their order shouldn't affect equality test. - unknown_fields = list(self._unknown_fields) - unknown_fields.sort() - other_unknown_fields = list(other._unknown_fields) - other_unknown_fields.sort() - - return unknown_fields == other_unknown_fields - - cls.__eq__ = __eq__ - - -def _AddStrMethod(message_descriptor, cls): - """Helper for _AddMessageMethods().""" - def __str__(self): - return text_format.MessageToString(self) - cls.__str__ = __str__ - - -def _AddUnicodeMethod(unused_message_descriptor, cls): - """Helper for _AddMessageMethods().""" - - def __unicode__(self): - return text_format.MessageToString(self, as_utf8=True).decode('utf-8') - cls.__unicode__ = __unicode__ - - -def _AddSetListenerMethod(cls): - """Helper for _AddMessageMethods().""" - def SetListener(self, listener): - if listener is None: - self._listener = message_listener_mod.NullMessageListener() - else: - self._listener = listener - cls._SetListener = SetListener - - -def _BytesForNonRepeatedElement(value, field_number, field_type): - """Returns the number of bytes needed to serialize a non-repeated element. - The returned byte count includes space for tag information and any - other additional space associated with serializing value. - - Args: - value: Value we're serializing. - field_number: Field number of this value. (Since the field number - is stored as part of a varint-encoded tag, this has an impact - on the total bytes required to serialize the value). - field_type: The type of the field. One of the TYPE_* constants - within FieldDescriptor. - """ - try: - fn = type_checkers.TYPE_TO_BYTE_SIZE_FN[field_type] - return fn(field_number, value) - except KeyError: - raise message_mod.EncodeError('Unrecognized field type: %d' % field_type) - - -def _AddByteSizeMethod(message_descriptor, cls): - """Helper for _AddMessageMethods().""" - - def ByteSize(self): - if not self._cached_byte_size_dirty: - return self._cached_byte_size - - size = 0 - for field_descriptor, field_value in self.ListFields(): - size += field_descriptor._sizer(field_value) - - for tag_bytes, value_bytes in self._unknown_fields: - size += len(tag_bytes) + len(value_bytes) - - self._cached_byte_size = size - self._cached_byte_size_dirty = False - self._listener_for_children.dirty = False - return size - - cls.ByteSize = ByteSize - - -def _AddSerializeToStringMethod(message_descriptor, cls): - """Helper for _AddMessageMethods().""" - - def SerializeToString(self): - # Check if the message has all of its required fields set. - errors = [] - if not self.IsInitialized(): - raise message_mod.EncodeError( - 'Message %s is missing required fields: %s' % ( - self.DESCRIPTOR.full_name, ','.join(self.FindInitializationErrors()))) - return self.SerializePartialToString() - cls.SerializeToString = SerializeToString - - -def _AddSerializePartialToStringMethod(message_descriptor, cls): - """Helper for _AddMessageMethods().""" - - def SerializePartialToString(self): - out = StringIO() - self._InternalSerialize(out.write) - return out.getvalue() - cls.SerializePartialToString = SerializePartialToString - - def InternalSerialize(self, write_bytes): - for field_descriptor, field_value in self.ListFields(): - field_descriptor._encoder(write_bytes, field_value) - for tag_bytes, value_bytes in self._unknown_fields: - write_bytes(tag_bytes) - write_bytes(value_bytes) - cls._InternalSerialize = InternalSerialize - - -def _AddMergeFromStringMethod(message_descriptor, cls): - """Helper for _AddMessageMethods().""" - def MergeFromString(self, serialized): - length = len(serialized) - try: - if self._InternalParse(serialized, 0, length) != length: - # The only reason _InternalParse would return early is if it - # encountered an end-group tag. - raise message_mod.DecodeError('Unexpected end-group tag.') - except IndexError: - raise message_mod.DecodeError('Truncated message.') - except struct.error, e: - raise message_mod.DecodeError(e) - return length # Return this for legacy reasons. - cls.MergeFromString = MergeFromString - - local_ReadTag = decoder.ReadTag - local_SkipField = decoder.SkipField - decoders_by_tag = cls._decoders_by_tag - - def InternalParse(self, buffer, pos, end): - self._Modified() - field_dict = self._fields - unknown_field_list = self._unknown_fields - while pos != end: - (tag_bytes, new_pos) = local_ReadTag(buffer, pos) - field_decoder = decoders_by_tag.get(tag_bytes) - if field_decoder is None: - value_start_pos = new_pos - new_pos = local_SkipField(buffer, new_pos, end, tag_bytes) - if new_pos == -1: - return pos - if not unknown_field_list: - unknown_field_list = self._unknown_fields = [] - unknown_field_list.append((tag_bytes, buffer[value_start_pos:new_pos])) - pos = new_pos - else: - pos = field_decoder(buffer, new_pos, end, self, field_dict) - return pos - cls._InternalParse = InternalParse - - -def _AddIsInitializedMethod(message_descriptor, cls): - """Adds the IsInitialized and FindInitializationError methods to the - protocol message class.""" - - required_fields = [field for field in message_descriptor.fields - if field.label == _FieldDescriptor.LABEL_REQUIRED] - - def IsInitialized(self, errors=None): - """Checks if all required fields of a message are set. - - Args: - errors: A list which, if provided, will be populated with the field - paths of all missing required fields. - - Returns: - True iff the specified message has all required fields set. - """ - - # Performance is critical so we avoid HasField() and ListFields(). - - for field in required_fields: - if (field not in self._fields or - (field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE and - not self._fields[field]._is_present_in_parent)): - if errors is not None: - errors.extend(self.FindInitializationErrors()) - return False - - for field, value in self._fields.iteritems(): - if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: - if field.label == _FieldDescriptor.LABEL_REPEATED: - for element in value: - if not element.IsInitialized(): - if errors is not None: - errors.extend(self.FindInitializationErrors()) - return False - elif value._is_present_in_parent and not value.IsInitialized(): - if errors is not None: - errors.extend(self.FindInitializationErrors()) - return False - - return True - - cls.IsInitialized = IsInitialized - - def FindInitializationErrors(self): - """Finds required fields which are not initialized. - - Returns: - A list of strings. Each string is a path to an uninitialized field from - the top-level message, e.g. "foo.bar[5].baz". - """ - - errors = [] # simplify things - - for field in required_fields: - if not self.HasField(field.name): - errors.append(field.name) - - for field, value in self.ListFields(): - if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: - if field.is_extension: - name = "(%s)" % field.full_name - else: - name = field.name - - if field.label == _FieldDescriptor.LABEL_REPEATED: - for i in xrange(len(value)): - element = value[i] - prefix = "%s[%d]." % (name, i) - sub_errors = element.FindInitializationErrors() - errors += [ prefix + error for error in sub_errors ] - else: - prefix = name + "." - sub_errors = value.FindInitializationErrors() - errors += [ prefix + error for error in sub_errors ] - - return errors - - cls.FindInitializationErrors = FindInitializationErrors - - -def _AddMergeFromMethod(cls): - LABEL_REPEATED = _FieldDescriptor.LABEL_REPEATED - CPPTYPE_MESSAGE = _FieldDescriptor.CPPTYPE_MESSAGE - - def MergeFrom(self, msg): - if not isinstance(msg, cls): - raise TypeError( - "Parameter to MergeFrom() must be instance of same class: " - "expected %s got %s." % (cls.__name__, type(msg).__name__)) - - assert msg is not self - self._Modified() - - fields = self._fields - - for field, value in msg._fields.iteritems(): - if field.label == LABEL_REPEATED: - field_value = fields.get(field) - if field_value is None: - # Construct a new object to represent this field. - field_value = field._default_constructor(self) - fields[field] = field_value - field_value.MergeFrom(value) - elif field.cpp_type == CPPTYPE_MESSAGE: - if value._is_present_in_parent: - field_value = fields.get(field) - if field_value is None: - # Construct a new object to represent this field. - field_value = field._default_constructor(self) - fields[field] = field_value - field_value.MergeFrom(value) - else: - self._fields[field] = value - - if msg._unknown_fields: - if not self._unknown_fields: - self._unknown_fields = [] - self._unknown_fields.extend(msg._unknown_fields) - - cls.MergeFrom = MergeFrom - - -def _AddMessageMethods(message_descriptor, cls): - """Adds implementations of all Message methods to cls.""" - _AddListFieldsMethod(message_descriptor, cls) - _AddHasFieldMethod(message_descriptor, cls) - _AddClearFieldMethod(message_descriptor, cls) - if message_descriptor.is_extendable: - _AddClearExtensionMethod(cls) - _AddHasExtensionMethod(cls) - _AddClearMethod(message_descriptor, cls) - _AddEqualsMethod(message_descriptor, cls) - _AddStrMethod(message_descriptor, cls) - _AddUnicodeMethod(message_descriptor, cls) - _AddSetListenerMethod(cls) - _AddByteSizeMethod(message_descriptor, cls) - _AddSerializeToStringMethod(message_descriptor, cls) - _AddSerializePartialToStringMethod(message_descriptor, cls) - _AddMergeFromStringMethod(message_descriptor, cls) - _AddIsInitializedMethod(message_descriptor, cls) - _AddMergeFromMethod(cls) - - -def _AddPrivateHelperMethods(cls): - """Adds implementation of private helper methods to cls.""" - - def Modified(self): - """Sets the _cached_byte_size_dirty bit to true, - and propagates this to our listener iff this was a state change. - """ - - # Note: Some callers check _cached_byte_size_dirty before calling - # _Modified() as an extra optimization. So, if this method is ever - # changed such that it does stuff even when _cached_byte_size_dirty is - # already true, the callers need to be updated. - if not self._cached_byte_size_dirty: - self._cached_byte_size_dirty = True - self._listener_for_children.dirty = True - self._is_present_in_parent = True - self._listener.Modified() - - cls._Modified = Modified - cls.SetInParent = Modified - - -class _Listener(object): - - """MessageListener implementation that a parent message registers with its - child message. - - In order to support semantics like: - - foo.bar.baz.qux = 23 - assert foo.HasField('bar') - - ...child objects must have back references to their parents. - This helper class is at the heart of this support. - """ - - def __init__(self, parent_message): - """Args: - parent_message: The message whose _Modified() method we should call when - we receive Modified() messages. - """ - # This listener establishes a back reference from a child (contained) object - # to its parent (containing) object. We make this a weak reference to avoid - # creating cyclic garbage when the client finishes with the 'parent' object - # in the tree. - if isinstance(parent_message, weakref.ProxyType): - self._parent_message_weakref = parent_message - else: - self._parent_message_weakref = weakref.proxy(parent_message) - - # As an optimization, we also indicate directly on the listener whether - # or not the parent message is dirty. This way we can avoid traversing - # up the tree in the common case. - self.dirty = False - - def Modified(self): - if self.dirty: - return - try: - # Propagate the signal to our parents iff this is the first field set. - self._parent_message_weakref._Modified() - except ReferenceError: - # We can get here if a client has kept a reference to a child object, - # and is now setting a field on it, but the child's parent has been - # garbage-collected. This is not an error. - pass - - -# TODO(robinson): Move elsewhere? This file is getting pretty ridiculous... -# TODO(robinson): Unify error handling of "unknown extension" crap. -# TODO(robinson): Support iteritems()-style iteration over all -# extensions with the "has" bits turned on? -class _ExtensionDict(object): - - """Dict-like container for supporting an indexable "Extensions" - field on proto instances. - - Note that in all cases we expect extension handles to be - FieldDescriptors. - """ - - def __init__(self, extended_message): - """extended_message: Message instance for which we are the Extensions dict. - """ - - self._extended_message = extended_message - - def __getitem__(self, extension_handle): - """Returns the current value of the given extension handle.""" - - _VerifyExtensionHandle(self._extended_message, extension_handle) - - result = self._extended_message._fields.get(extension_handle) - if result is not None: - return result - - if extension_handle.label == _FieldDescriptor.LABEL_REPEATED: - result = extension_handle._default_constructor(self._extended_message) - elif extension_handle.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: - result = extension_handle.message_type._concrete_class() - try: - result._SetListener(self._extended_message._listener_for_children) - except ReferenceError: - pass - else: - # Singular scalar -- just return the default without inserting into the - # dict. - return extension_handle.default_value - - # Atomically check if another thread has preempted us and, if not, swap - # in the new object we just created. If someone has preempted us, we - # take that object and discard ours. - # WARNING: We are relying on setdefault() being atomic. This is true - # in CPython but we haven't investigated others. This warning appears - # in several other locations in this file. - result = self._extended_message._fields.setdefault( - extension_handle, result) - - return result - - def __eq__(self, other): - if not isinstance(other, self.__class__): - return False - - my_fields = self._extended_message.ListFields() - other_fields = other._extended_message.ListFields() - - # Get rid of non-extension fields. - my_fields = [ field for field in my_fields if field.is_extension ] - other_fields = [ field for field in other_fields if field.is_extension ] - - return my_fields == other_fields - - def __ne__(self, other): - return not self == other - - def __hash__(self): - raise TypeError('unhashable object') - - # Note that this is only meaningful for non-repeated, scalar extension - # fields. Note also that we may have to call _Modified() when we do - # successfully set a field this way, to set any necssary "has" bits in the - # ancestors of the extended message. - def __setitem__(self, extension_handle, value): - """If extension_handle specifies a non-repeated, scalar extension - field, sets the value of that field. - """ - - _VerifyExtensionHandle(self._extended_message, extension_handle) - - if (extension_handle.label == _FieldDescriptor.LABEL_REPEATED or - extension_handle.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE): - raise TypeError( - 'Cannot assign to extension "%s" because it is a repeated or ' - 'composite type.' % extension_handle.full_name) - - # It's slightly wasteful to lookup the type checker each time, - # but we expect this to be a vanishingly uncommon case anyway. - type_checker = type_checkers.GetTypeChecker( - extension_handle.cpp_type, extension_handle.type) - type_checker.CheckValue(value) - self._extended_message._fields[extension_handle] = value - self._extended_message._Modified() - - def _FindExtensionByName(self, name): - """Tries to find a known extension with the specified name. - - Args: - name: Extension full name. - - Returns: - Extension field descriptor. - """ - return self._extended_message._extensions_by_name.get(name, None) diff --git a/generator/google/protobuf/internal/type_checkers.py b/generator/google/protobuf/internal/type_checkers.py deleted file mode 100644 index 2b3cd4d..0000000 --- a/generator/google/protobuf/internal/type_checkers.py +++ /dev/null @@ -1,286 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Provides type checking routines. - -This module defines type checking utilities in the forms of dictionaries: - -VALUE_CHECKERS: A dictionary of field types and a value validation object. -TYPE_TO_BYTE_SIZE_FN: A dictionary with field types and a size computing - function. -TYPE_TO_SERIALIZE_METHOD: A dictionary with field types and serialization - function. -FIELD_TYPE_TO_WIRE_TYPE: A dictionary with field typed and their - coresponding wire types. -TYPE_TO_DESERIALIZE_METHOD: A dictionary with field types and deserialization - function. -""" - -__author__ = 'robinson@google.com (Will Robinson)' - -from google.protobuf.internal import decoder -from google.protobuf.internal import encoder -from google.protobuf.internal import wire_format -from google.protobuf import descriptor - -_FieldDescriptor = descriptor.FieldDescriptor - - -def GetTypeChecker(cpp_type, field_type): - """Returns a type checker for a message field of the specified types. - - Args: - cpp_type: C++ type of the field (see descriptor.py). - field_type: Protocol message field type (see descriptor.py). - - Returns: - An instance of TypeChecker which can be used to verify the types - of values assigned to a field of the specified type. - """ - if (cpp_type == _FieldDescriptor.CPPTYPE_STRING and - field_type == _FieldDescriptor.TYPE_STRING): - return UnicodeValueChecker() - return _VALUE_CHECKERS[cpp_type] - - -# None of the typecheckers below make any attempt to guard against people -# subclassing builtin types and doing weird things. We're not trying to -# protect against malicious clients here, just people accidentally shooting -# themselves in the foot in obvious ways. - -class TypeChecker(object): - - """Type checker used to catch type errors as early as possible - when the client is setting scalar fields in protocol messages. - """ - - def __init__(self, *acceptable_types): - self._acceptable_types = acceptable_types - - def CheckValue(self, proposed_value): - if not isinstance(proposed_value, self._acceptable_types): - message = ('%.1024r has type %s, but expected one of: %s' % - (proposed_value, type(proposed_value), self._acceptable_types)) - raise TypeError(message) - - -# IntValueChecker and its subclasses perform integer type-checks -# and bounds-checks. -class IntValueChecker(object): - - """Checker used for integer fields. Performs type-check and range check.""" - - def CheckValue(self, proposed_value): - if not isinstance(proposed_value, (int, long)): - message = ('%.1024r has type %s, but expected one of: %s' % - (proposed_value, type(proposed_value), (int, long))) - raise TypeError(message) - if not self._MIN <= proposed_value <= self._MAX: - raise ValueError('Value out of range: %d' % proposed_value) - - -class UnicodeValueChecker(object): - - """Checker used for string fields.""" - - def CheckValue(self, proposed_value): - if not isinstance(proposed_value, (str, unicode)): - message = ('%.1024r has type %s, but expected one of: %s' % - (proposed_value, type(proposed_value), (str, unicode))) - raise TypeError(message) - - # If the value is of type 'str' make sure that it is in 7-bit ASCII - # encoding. - if isinstance(proposed_value, str): - try: - unicode(proposed_value, 'ascii') - except UnicodeDecodeError: - raise ValueError('%.1024r has type str, but isn\'t in 7-bit ASCII ' - 'encoding. Non-ASCII strings must be converted to ' - 'unicode objects before being added.' % - (proposed_value)) - - -class Int32ValueChecker(IntValueChecker): - # We're sure to use ints instead of longs here since comparison may be more - # efficient. - _MIN = -2147483648 - _MAX = 2147483647 - - -class Uint32ValueChecker(IntValueChecker): - _MIN = 0 - _MAX = (1 << 32) - 1 - - -class Int64ValueChecker(IntValueChecker): - _MIN = -(1 << 63) - _MAX = (1 << 63) - 1 - - -class Uint64ValueChecker(IntValueChecker): - _MIN = 0 - _MAX = (1 << 64) - 1 - - -# Type-checkers for all scalar CPPTYPEs. -_VALUE_CHECKERS = { - _FieldDescriptor.CPPTYPE_INT32: Int32ValueChecker(), - _FieldDescriptor.CPPTYPE_INT64: Int64ValueChecker(), - _FieldDescriptor.CPPTYPE_UINT32: Uint32ValueChecker(), - _FieldDescriptor.CPPTYPE_UINT64: Uint64ValueChecker(), - _FieldDescriptor.CPPTYPE_DOUBLE: TypeChecker( - float, int, long), - _FieldDescriptor.CPPTYPE_FLOAT: TypeChecker( - float, int, long), - _FieldDescriptor.CPPTYPE_BOOL: TypeChecker(bool, int), - _FieldDescriptor.CPPTYPE_ENUM: Int32ValueChecker(), - _FieldDescriptor.CPPTYPE_STRING: TypeChecker(str), - } - - -# Map from field type to a function F, such that F(field_num, value) -# gives the total byte size for a value of the given type. This -# byte size includes tag information and any other additional space -# associated with serializing "value". -TYPE_TO_BYTE_SIZE_FN = { - _FieldDescriptor.TYPE_DOUBLE: wire_format.DoubleByteSize, - _FieldDescriptor.TYPE_FLOAT: wire_format.FloatByteSize, - _FieldDescriptor.TYPE_INT64: wire_format.Int64ByteSize, - _FieldDescriptor.TYPE_UINT64: wire_format.UInt64ByteSize, - _FieldDescriptor.TYPE_INT32: wire_format.Int32ByteSize, - _FieldDescriptor.TYPE_FIXED64: wire_format.Fixed64ByteSize, - _FieldDescriptor.TYPE_FIXED32: wire_format.Fixed32ByteSize, - _FieldDescriptor.TYPE_BOOL: wire_format.BoolByteSize, - _FieldDescriptor.TYPE_STRING: wire_format.StringByteSize, - _FieldDescriptor.TYPE_GROUP: wire_format.GroupByteSize, - _FieldDescriptor.TYPE_MESSAGE: wire_format.MessageByteSize, - _FieldDescriptor.TYPE_BYTES: wire_format.BytesByteSize, - _FieldDescriptor.TYPE_UINT32: wire_format.UInt32ByteSize, - _FieldDescriptor.TYPE_ENUM: wire_format.EnumByteSize, - _FieldDescriptor.TYPE_SFIXED32: wire_format.SFixed32ByteSize, - _FieldDescriptor.TYPE_SFIXED64: wire_format.SFixed64ByteSize, - _FieldDescriptor.TYPE_SINT32: wire_format.SInt32ByteSize, - _FieldDescriptor.TYPE_SINT64: wire_format.SInt64ByteSize - } - - -# Maps from field types to encoder constructors. -TYPE_TO_ENCODER = { - _FieldDescriptor.TYPE_DOUBLE: encoder.DoubleEncoder, - _FieldDescriptor.TYPE_FLOAT: encoder.FloatEncoder, - _FieldDescriptor.TYPE_INT64: encoder.Int64Encoder, - _FieldDescriptor.TYPE_UINT64: encoder.UInt64Encoder, - _FieldDescriptor.TYPE_INT32: encoder.Int32Encoder, - _FieldDescriptor.TYPE_FIXED64: encoder.Fixed64Encoder, - _FieldDescriptor.TYPE_FIXED32: encoder.Fixed32Encoder, - _FieldDescriptor.TYPE_BOOL: encoder.BoolEncoder, - _FieldDescriptor.TYPE_STRING: encoder.StringEncoder, - _FieldDescriptor.TYPE_GROUP: encoder.GroupEncoder, - _FieldDescriptor.TYPE_MESSAGE: encoder.MessageEncoder, - _FieldDescriptor.TYPE_BYTES: encoder.BytesEncoder, - _FieldDescriptor.TYPE_UINT32: encoder.UInt32Encoder, - _FieldDescriptor.TYPE_ENUM: encoder.EnumEncoder, - _FieldDescriptor.TYPE_SFIXED32: encoder.SFixed32Encoder, - _FieldDescriptor.TYPE_SFIXED64: encoder.SFixed64Encoder, - _FieldDescriptor.TYPE_SINT32: encoder.SInt32Encoder, - _FieldDescriptor.TYPE_SINT64: encoder.SInt64Encoder, - } - - -# Maps from field types to sizer constructors. -TYPE_TO_SIZER = { - _FieldDescriptor.TYPE_DOUBLE: encoder.DoubleSizer, - _FieldDescriptor.TYPE_FLOAT: encoder.FloatSizer, - _FieldDescriptor.TYPE_INT64: encoder.Int64Sizer, - _FieldDescriptor.TYPE_UINT64: encoder.UInt64Sizer, - _FieldDescriptor.TYPE_INT32: encoder.Int32Sizer, - _FieldDescriptor.TYPE_FIXED64: encoder.Fixed64Sizer, - _FieldDescriptor.TYPE_FIXED32: encoder.Fixed32Sizer, - _FieldDescriptor.TYPE_BOOL: encoder.BoolSizer, - _FieldDescriptor.TYPE_STRING: encoder.StringSizer, - _FieldDescriptor.TYPE_GROUP: encoder.GroupSizer, - _FieldDescriptor.TYPE_MESSAGE: encoder.MessageSizer, - _FieldDescriptor.TYPE_BYTES: encoder.BytesSizer, - _FieldDescriptor.TYPE_UINT32: encoder.UInt32Sizer, - _FieldDescriptor.TYPE_ENUM: encoder.EnumSizer, - _FieldDescriptor.TYPE_SFIXED32: encoder.SFixed32Sizer, - _FieldDescriptor.TYPE_SFIXED64: encoder.SFixed64Sizer, - _FieldDescriptor.TYPE_SINT32: encoder.SInt32Sizer, - _FieldDescriptor.TYPE_SINT64: encoder.SInt64Sizer, - } - - -# Maps from field type to a decoder constructor. -TYPE_TO_DECODER = { - _FieldDescriptor.TYPE_DOUBLE: decoder.DoubleDecoder, - _FieldDescriptor.TYPE_FLOAT: decoder.FloatDecoder, - _FieldDescriptor.TYPE_INT64: decoder.Int64Decoder, - _FieldDescriptor.TYPE_UINT64: decoder.UInt64Decoder, - _FieldDescriptor.TYPE_INT32: decoder.Int32Decoder, - _FieldDescriptor.TYPE_FIXED64: decoder.Fixed64Decoder, - _FieldDescriptor.TYPE_FIXED32: decoder.Fixed32Decoder, - _FieldDescriptor.TYPE_BOOL: decoder.BoolDecoder, - _FieldDescriptor.TYPE_STRING: decoder.StringDecoder, - _FieldDescriptor.TYPE_GROUP: decoder.GroupDecoder, - _FieldDescriptor.TYPE_MESSAGE: decoder.MessageDecoder, - _FieldDescriptor.TYPE_BYTES: decoder.BytesDecoder, - _FieldDescriptor.TYPE_UINT32: decoder.UInt32Decoder, - _FieldDescriptor.TYPE_ENUM: decoder.EnumDecoder, - _FieldDescriptor.TYPE_SFIXED32: decoder.SFixed32Decoder, - _FieldDescriptor.TYPE_SFIXED64: decoder.SFixed64Decoder, - _FieldDescriptor.TYPE_SINT32: decoder.SInt32Decoder, - _FieldDescriptor.TYPE_SINT64: decoder.SInt64Decoder, - } - -# Maps from field type to expected wiretype. -FIELD_TYPE_TO_WIRE_TYPE = { - _FieldDescriptor.TYPE_DOUBLE: wire_format.WIRETYPE_FIXED64, - _FieldDescriptor.TYPE_FLOAT: wire_format.WIRETYPE_FIXED32, - _FieldDescriptor.TYPE_INT64: wire_format.WIRETYPE_VARINT, - _FieldDescriptor.TYPE_UINT64: wire_format.WIRETYPE_VARINT, - _FieldDescriptor.TYPE_INT32: wire_format.WIRETYPE_VARINT, - _FieldDescriptor.TYPE_FIXED64: wire_format.WIRETYPE_FIXED64, - _FieldDescriptor.TYPE_FIXED32: wire_format.WIRETYPE_FIXED32, - _FieldDescriptor.TYPE_BOOL: wire_format.WIRETYPE_VARINT, - _FieldDescriptor.TYPE_STRING: - wire_format.WIRETYPE_LENGTH_DELIMITED, - _FieldDescriptor.TYPE_GROUP: wire_format.WIRETYPE_START_GROUP, - _FieldDescriptor.TYPE_MESSAGE: - wire_format.WIRETYPE_LENGTH_DELIMITED, - _FieldDescriptor.TYPE_BYTES: - wire_format.WIRETYPE_LENGTH_DELIMITED, - _FieldDescriptor.TYPE_UINT32: wire_format.WIRETYPE_VARINT, - _FieldDescriptor.TYPE_ENUM: wire_format.WIRETYPE_VARINT, - _FieldDescriptor.TYPE_SFIXED32: wire_format.WIRETYPE_FIXED32, - _FieldDescriptor.TYPE_SFIXED64: wire_format.WIRETYPE_FIXED64, - _FieldDescriptor.TYPE_SINT32: wire_format.WIRETYPE_VARINT, - _FieldDescriptor.TYPE_SINT64: wire_format.WIRETYPE_VARINT, - } diff --git a/generator/google/protobuf/internal/wire_format.py b/generator/google/protobuf/internal/wire_format.py deleted file mode 100644 index c941fe1..0000000 --- a/generator/google/protobuf/internal/wire_format.py +++ /dev/null @@ -1,268 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Constants and static functions to support protocol buffer wire format.""" - -__author__ = 'robinson@google.com (Will Robinson)' - -import struct -from google.protobuf import descriptor -from google.protobuf import message - - -TAG_TYPE_BITS = 3 # Number of bits used to hold type info in a proto tag. -TAG_TYPE_MASK = (1 << TAG_TYPE_BITS) - 1 # 0x7 - -# These numbers identify the wire type of a protocol buffer value. -# We use the least-significant TAG_TYPE_BITS bits of the varint-encoded -# tag-and-type to store one of these WIRETYPE_* constants. -# These values must match WireType enum in google/protobuf/wire_format.h. -WIRETYPE_VARINT = 0 -WIRETYPE_FIXED64 = 1 -WIRETYPE_LENGTH_DELIMITED = 2 -WIRETYPE_START_GROUP = 3 -WIRETYPE_END_GROUP = 4 -WIRETYPE_FIXED32 = 5 -_WIRETYPE_MAX = 5 - - -# Bounds for various integer types. -INT32_MAX = int((1 << 31) - 1) -INT32_MIN = int(-(1 << 31)) -UINT32_MAX = (1 << 32) - 1 - -INT64_MAX = (1 << 63) - 1 -INT64_MIN = -(1 << 63) -UINT64_MAX = (1 << 64) - 1 - -# "struct" format strings that will encode/decode the specified formats. -FORMAT_UINT32_LITTLE_ENDIAN = '<I' -FORMAT_UINT64_LITTLE_ENDIAN = '<Q' -FORMAT_FLOAT_LITTLE_ENDIAN = '<f' -FORMAT_DOUBLE_LITTLE_ENDIAN = '<d' - - -# We'll have to provide alternate implementations of AppendLittleEndian*() on -# any architectures where these checks fail. -if struct.calcsize(FORMAT_UINT32_LITTLE_ENDIAN) != 4: - raise AssertionError('Format "I" is not a 32-bit number.') -if struct.calcsize(FORMAT_UINT64_LITTLE_ENDIAN) != 8: - raise AssertionError('Format "Q" is not a 64-bit number.') - - -def PackTag(field_number, wire_type): - """Returns an unsigned 32-bit integer that encodes the field number and - wire type information in standard protocol message wire format. - - Args: - field_number: Expected to be an integer in the range [1, 1 << 29) - wire_type: One of the WIRETYPE_* constants. - """ - if not 0 <= wire_type <= _WIRETYPE_MAX: - raise message.EncodeError('Unknown wire type: %d' % wire_type) - return (field_number << TAG_TYPE_BITS) | wire_type - - -def UnpackTag(tag): - """The inverse of PackTag(). Given an unsigned 32-bit number, - returns a (field_number, wire_type) tuple. - """ - return (tag >> TAG_TYPE_BITS), (tag & TAG_TYPE_MASK) - - -def ZigZagEncode(value): - """ZigZag Transform: Encodes signed integers so that they can be - effectively used with varint encoding. See wire_format.h for - more details. - """ - if value >= 0: - return value << 1 - return (value << 1) ^ (~0) - - -def ZigZagDecode(value): - """Inverse of ZigZagEncode().""" - if not value & 0x1: - return value >> 1 - return (value >> 1) ^ (~0) - - - -# The *ByteSize() functions below return the number of bytes required to -# serialize "field number + type" information and then serialize the value. - - -def Int32ByteSize(field_number, int32): - return Int64ByteSize(field_number, int32) - - -def Int32ByteSizeNoTag(int32): - return _VarUInt64ByteSizeNoTag(0xffffffffffffffff & int32) - - -def Int64ByteSize(field_number, int64): - # Have to convert to uint before calling UInt64ByteSize(). - return UInt64ByteSize(field_number, 0xffffffffffffffff & int64) - - -def UInt32ByteSize(field_number, uint32): - return UInt64ByteSize(field_number, uint32) - - -def UInt64ByteSize(field_number, uint64): - return TagByteSize(field_number) + _VarUInt64ByteSizeNoTag(uint64) - - -def SInt32ByteSize(field_number, int32): - return UInt32ByteSize(field_number, ZigZagEncode(int32)) - - -def SInt64ByteSize(field_number, int64): - return UInt64ByteSize(field_number, ZigZagEncode(int64)) - - -def Fixed32ByteSize(field_number, fixed32): - return TagByteSize(field_number) + 4 - - -def Fixed64ByteSize(field_number, fixed64): - return TagByteSize(field_number) + 8 - - -def SFixed32ByteSize(field_number, sfixed32): - return TagByteSize(field_number) + 4 - - -def SFixed64ByteSize(field_number, sfixed64): - return TagByteSize(field_number) + 8 - - -def FloatByteSize(field_number, flt): - return TagByteSize(field_number) + 4 - - -def DoubleByteSize(field_number, double): - return TagByteSize(field_number) + 8 - - -def BoolByteSize(field_number, b): - return TagByteSize(field_number) + 1 - - -def EnumByteSize(field_number, enum): - return UInt32ByteSize(field_number, enum) - - -def StringByteSize(field_number, string): - return BytesByteSize(field_number, string.encode('utf-8')) - - -def BytesByteSize(field_number, b): - return (TagByteSize(field_number) - + _VarUInt64ByteSizeNoTag(len(b)) - + len(b)) - - -def GroupByteSize(field_number, message): - return (2 * TagByteSize(field_number) # START and END group. - + message.ByteSize()) - - -def MessageByteSize(field_number, message): - return (TagByteSize(field_number) - + _VarUInt64ByteSizeNoTag(message.ByteSize()) - + message.ByteSize()) - - -def MessageSetItemByteSize(field_number, msg): - # First compute the sizes of the tags. - # There are 2 tags for the beginning and ending of the repeated group, that - # is field number 1, one with field number 2 (type_id) and one with field - # number 3 (message). - total_size = (2 * TagByteSize(1) + TagByteSize(2) + TagByteSize(3)) - - # Add the number of bytes for type_id. - total_size += _VarUInt64ByteSizeNoTag(field_number) - - message_size = msg.ByteSize() - - # The number of bytes for encoding the length of the message. - total_size += _VarUInt64ByteSizeNoTag(message_size) - - # The size of the message. - total_size += message_size - return total_size - - -def TagByteSize(field_number): - """Returns the bytes required to serialize a tag with this field number.""" - # Just pass in type 0, since the type won't affect the tag+type size. - return _VarUInt64ByteSizeNoTag(PackTag(field_number, 0)) - - -# Private helper function for the *ByteSize() functions above. - -def _VarUInt64ByteSizeNoTag(uint64): - """Returns the number of bytes required to serialize a single varint - using boundary value comparisons. (unrolled loop optimization -WPierce) - uint64 must be unsigned. - """ - if uint64 <= 0x7f: return 1 - if uint64 <= 0x3fff: return 2 - if uint64 <= 0x1fffff: return 3 - if uint64 <= 0xfffffff: return 4 - if uint64 <= 0x7ffffffff: return 5 - if uint64 <= 0x3ffffffffff: return 6 - if uint64 <= 0x1ffffffffffff: return 7 - if uint64 <= 0xffffffffffffff: return 8 - if uint64 <= 0x7fffffffffffffff: return 9 - if uint64 > UINT64_MAX: - raise message.EncodeError('Value out of range: %d' % uint64) - return 10 - - -NON_PACKABLE_TYPES = ( - descriptor.FieldDescriptor.TYPE_STRING, - descriptor.FieldDescriptor.TYPE_GROUP, - descriptor.FieldDescriptor.TYPE_MESSAGE, - descriptor.FieldDescriptor.TYPE_BYTES -) - - -def IsTypePackable(field_type): - """Return true iff packable = true is valid for fields of this type. - - Args: - field_type: a FieldDescriptor::Type value. - - Returns: - True iff fields of this type are packable. - """ - return field_type not in NON_PACKABLE_TYPES diff --git a/generator/google/protobuf/message.py b/generator/google/protobuf/message.py deleted file mode 100644 index 6ec2f8b..0000000 --- a/generator/google/protobuf/message.py +++ /dev/null @@ -1,280 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# TODO(robinson): We should just make these methods all "pure-virtual" and move -# all implementation out, into reflection.py for now. - - -"""Contains an abstract base class for protocol messages.""" - -__author__ = 'robinson@google.com (Will Robinson)' - - -class Error(Exception): pass -class DecodeError(Error): pass -class EncodeError(Error): pass - - -class Message(object): - - """Abstract base class for protocol messages. - - Protocol message classes are almost always generated by the protocol - compiler. These generated types subclass Message and implement the methods - shown below. - - TODO(robinson): Link to an HTML document here. - - TODO(robinson): Document that instances of this class will also - have an Extensions attribute with __getitem__ and __setitem__. - Again, not sure how to best convey this. - - TODO(robinson): Document that the class must also have a static - RegisterExtension(extension_field) method. - Not sure how to best express at this point. - """ - - # TODO(robinson): Document these fields and methods. - - __slots__ = [] - - DESCRIPTOR = None - - def __deepcopy__(self, memo=None): - clone = type(self)() - clone.MergeFrom(self) - return clone - - def __eq__(self, other_msg): - """Recursively compares two messages by value and structure.""" - raise NotImplementedError - - def __ne__(self, other_msg): - # Can't just say self != other_msg, since that would infinitely recurse. :) - return not self == other_msg - - def __hash__(self): - raise TypeError('unhashable object') - - def __str__(self): - """Outputs a human-readable representation of the message.""" - raise NotImplementedError - - def __unicode__(self): - """Outputs a human-readable representation of the message.""" - raise NotImplementedError - - def MergeFrom(self, other_msg): - """Merges the contents of the specified message into current message. - - This method merges the contents of the specified message into the current - message. Singular fields that are set in the specified message overwrite - the corresponding fields in the current message. Repeated fields are - appended. Singular sub-messages and groups are recursively merged. - - Args: - other_msg: Message to merge into the current message. - """ - raise NotImplementedError - - def CopyFrom(self, other_msg): - """Copies the content of the specified message into the current message. - - The method clears the current message and then merges the specified - message using MergeFrom. - - Args: - other_msg: Message to copy into the current one. - """ - if self is other_msg: - return - self.Clear() - self.MergeFrom(other_msg) - - def Clear(self): - """Clears all data that was set in the message.""" - raise NotImplementedError - - def SetInParent(self): - """Mark this as present in the parent. - - This normally happens automatically when you assign a field of a - sub-message, but sometimes you want to make the sub-message - present while keeping it empty. If you find yourself using this, - you may want to reconsider your design.""" - raise NotImplementedError - - def IsInitialized(self): - """Checks if the message is initialized. - - Returns: - The method returns True if the message is initialized (i.e. all of its - required fields are set). - """ - raise NotImplementedError - - # TODO(robinson): MergeFromString() should probably return None and be - # implemented in terms of a helper that returns the # of bytes read. Our - # deserialization routines would use the helper when recursively - # deserializing, but the end user would almost always just want the no-return - # MergeFromString(). - - def MergeFromString(self, serialized): - """Merges serialized protocol buffer data into this message. - - When we find a field in |serialized| that is already present - in this message: - - If it's a "repeated" field, we append to the end of our list. - - Else, if it's a scalar, we overwrite our field. - - Else, (it's a nonrepeated composite), we recursively merge - into the existing composite. - - TODO(robinson): Document handling of unknown fields. - - Args: - serialized: Any object that allows us to call buffer(serialized) - to access a string of bytes using the buffer interface. - - TODO(robinson): When we switch to a helper, this will return None. - - Returns: - The number of bytes read from |serialized|. - For non-group messages, this will always be len(serialized), - but for messages which are actually groups, this will - generally be less than len(serialized), since we must - stop when we reach an END_GROUP tag. Note that if - we *do* stop because of an END_GROUP tag, the number - of bytes returned does not include the bytes - for the END_GROUP tag information. - """ - raise NotImplementedError - - def ParseFromString(self, serialized): - """Like MergeFromString(), except we clear the object first.""" - self.Clear() - self.MergeFromString(serialized) - - def SerializeToString(self): - """Serializes the protocol message to a binary string. - - Returns: - A binary string representation of the message if all of the required - fields in the message are set (i.e. the message is initialized). - - Raises: - message.EncodeError if the message isn't initialized. - """ - raise NotImplementedError - - def SerializePartialToString(self): - """Serializes the protocol message to a binary string. - - This method is similar to SerializeToString but doesn't check if the - message is initialized. - - Returns: - A string representation of the partial message. - """ - raise NotImplementedError - - # TODO(robinson): Decide whether we like these better - # than auto-generated has_foo() and clear_foo() methods - # on the instances themselves. This way is less consistent - # with C++, but it makes reflection-type access easier and - # reduces the number of magically autogenerated things. - # - # TODO(robinson): Be sure to document (and test) exactly - # which field names are accepted here. Are we case-sensitive? - # What do we do with fields that share names with Python keywords - # like 'lambda' and 'yield'? - # - # nnorwitz says: - # """ - # Typically (in python), an underscore is appended to names that are - # keywords. So they would become lambda_ or yield_. - # """ - def ListFields(self): - """Returns a list of (FieldDescriptor, value) tuples for all - fields in the message which are not empty. A singular field is non-empty - if HasField() would return true, and a repeated field is non-empty if - it contains at least one element. The fields are ordered by field - number""" - raise NotImplementedError - - def HasField(self, field_name): - """Checks if a certain field is set for the message. Note if the - field_name is not defined in the message descriptor, ValueError will be - raised.""" - raise NotImplementedError - - def ClearField(self, field_name): - raise NotImplementedError - - def HasExtension(self, extension_handle): - raise NotImplementedError - - def ClearExtension(self, extension_handle): - raise NotImplementedError - - def ByteSize(self): - """Returns the serialized size of this message. - Recursively calls ByteSize() on all contained messages. - """ - raise NotImplementedError - - def _SetListener(self, message_listener): - """Internal method used by the protocol message implementation. - Clients should not call this directly. - - Sets a listener that this message will call on certain state transitions. - - The purpose of this method is to register back-edges from children to - parents at runtime, for the purpose of setting "has" bits and - byte-size-dirty bits in the parent and ancestor objects whenever a child or - descendant object is modified. - - If the client wants to disconnect this Message from the object tree, she - explicitly sets callback to None. - - If message_listener is None, unregisters any existing listener. Otherwise, - message_listener must implement the MessageListener interface in - internal/message_listener.py, and we discard any listener registered - via a previous _SetListener() call. - """ - raise NotImplementedError - - def __getstate__(self): - """Support the pickle protocol.""" - return dict(serialized=self.SerializePartialToString()) - - def __setstate__(self, state): - """Support the pickle protocol.""" - self.__init__() - self.ParseFromString(state['serialized']) diff --git a/generator/google/protobuf/message_factory.py b/generator/google/protobuf/message_factory.py deleted file mode 100644 index 36e2fef..0000000 --- a/generator/google/protobuf/message_factory.py +++ /dev/null @@ -1,113 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Provides a factory class for generating dynamic messages.""" - -__author__ = 'matthewtoia@google.com (Matt Toia)' - -from google.protobuf import descriptor_database -from google.protobuf import descriptor_pool -from google.protobuf import message -from google.protobuf import reflection - - -class MessageFactory(object): - """Factory for creating Proto2 messages from descriptors in a pool.""" - - def __init__(self): - """Initializes a new factory.""" - self._classes = {} - - def GetPrototype(self, descriptor): - """Builds a proto2 message class based on the passed in descriptor. - - Passing a descriptor with a fully qualified name matching a previous - invocation will cause the same class to be returned. - - Args: - descriptor: The descriptor to build from. - - Returns: - A class describing the passed in descriptor. - """ - - if descriptor.full_name not in self._classes: - result_class = reflection.GeneratedProtocolMessageType( - descriptor.name.encode('ascii', 'ignore'), - (message.Message,), - {'DESCRIPTOR': descriptor}) - self._classes[descriptor.full_name] = result_class - for field in descriptor.fields: - if field.message_type: - self.GetPrototype(field.message_type) - return self._classes[descriptor.full_name] - - -_DB = descriptor_database.DescriptorDatabase() -_POOL = descriptor_pool.DescriptorPool(_DB) -_FACTORY = MessageFactory() - - -def GetMessages(file_protos): - """Builds a dictionary of all the messages available in a set of files. - - Args: - file_protos: A sequence of file protos to build messages out of. - - Returns: - A dictionary containing all the message types in the files mapping the - fully qualified name to a Message subclass for the descriptor. - """ - - result = {} - for file_proto in file_protos: - _DB.Add(file_proto) - for file_proto in file_protos: - for desc in _GetAllDescriptors(file_proto.message_type, file_proto.package): - result[desc.full_name] = _FACTORY.GetPrototype(desc) - return result - - -def _GetAllDescriptors(desc_protos, package): - """Gets all levels of nested message types as a flattened list of descriptors. - - Args: - desc_protos: The descriptor protos to process. - package: The package where the protos are defined. - - Yields: - Each message descriptor for each nested type. - """ - - for desc_proto in desc_protos: - name = '.'.join((package, desc_proto.name)) - yield _POOL.FindMessageTypeByName(name) - for nested_desc in _GetAllDescriptors(desc_proto.nested_type, name): - yield nested_desc diff --git a/generator/google/protobuf/reflection.py b/generator/google/protobuf/reflection.py deleted file mode 100644 index 9570fd5..0000000 --- a/generator/google/protobuf/reflection.py +++ /dev/null @@ -1,169 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -# This code is meant to work on Python 2.4 and above only. - -"""Contains a metaclass and helper functions used to create -protocol message classes from Descriptor objects at runtime. - -Recall that a metaclass is the "type" of a class. -(A class is to a metaclass what an instance is to a class.) - -In this case, we use the GeneratedProtocolMessageType metaclass -to inject all the useful functionality into the classes -output by the protocol compiler at compile-time. - -The upshot of all this is that the real implementation -details for ALL pure-Python protocol buffers are *here in -this file*. -""" - -__author__ = 'robinson@google.com (Will Robinson)' - - -from google.protobuf.internal import api_implementation -from google.protobuf import descriptor as descriptor_mod -from google.protobuf import message - -_FieldDescriptor = descriptor_mod.FieldDescriptor - - -if api_implementation.Type() == 'cpp': - if api_implementation.Version() == 2: - from google.protobuf.internal.cpp import cpp_message - _NewMessage = cpp_message.NewMessage - _InitMessage = cpp_message.InitMessage - else: - from google.protobuf.internal import cpp_message - _NewMessage = cpp_message.NewMessage - _InitMessage = cpp_message.InitMessage -else: - from google.protobuf.internal import python_message - _NewMessage = python_message.NewMessage - _InitMessage = python_message.InitMessage - - -class GeneratedProtocolMessageType(type): - - """Metaclass for protocol message classes created at runtime from Descriptors. - - We add implementations for all methods described in the Message class. We - also create properties to allow getting/setting all fields in the protocol - message. Finally, we create slots to prevent users from accidentally - "setting" nonexistent fields in the protocol message, which then wouldn't get - serialized / deserialized properly. - - The protocol compiler currently uses this metaclass to create protocol - message classes at runtime. Clients can also manually create their own - classes at runtime, as in this example: - - mydescriptor = Descriptor(.....) - class MyProtoClass(Message): - __metaclass__ = GeneratedProtocolMessageType - DESCRIPTOR = mydescriptor - myproto_instance = MyProtoClass() - myproto.foo_field = 23 - ... - """ - - # Must be consistent with the protocol-compiler code in - # proto2/compiler/internal/generator.*. - _DESCRIPTOR_KEY = 'DESCRIPTOR' - - def __new__(cls, name, bases, dictionary): - """Custom allocation for runtime-generated class types. - - We override __new__ because this is apparently the only place - where we can meaningfully set __slots__ on the class we're creating(?). - (The interplay between metaclasses and slots is not very well-documented). - - Args: - name: Name of the class (ignored, but required by the - metaclass protocol). - bases: Base classes of the class we're constructing. - (Should be message.Message). We ignore this field, but - it's required by the metaclass protocol - dictionary: The class dictionary of the class we're - constructing. dictionary[_DESCRIPTOR_KEY] must contain - a Descriptor object describing this protocol message - type. - - Returns: - Newly-allocated class. - """ - descriptor = dictionary[GeneratedProtocolMessageType._DESCRIPTOR_KEY] - bases = _NewMessage(bases, descriptor, dictionary) - superclass = super(GeneratedProtocolMessageType, cls) - - new_class = superclass.__new__(cls, name, bases, dictionary) - setattr(descriptor, '_concrete_class', new_class) - return new_class - - def __init__(cls, name, bases, dictionary): - """Here we perform the majority of our work on the class. - We add enum getters, an __init__ method, implementations - of all Message methods, and properties for all fields - in the protocol type. - - Args: - name: Name of the class (ignored, but required by the - metaclass protocol). - bases: Base classes of the class we're constructing. - (Should be message.Message). We ignore this field, but - it's required by the metaclass protocol - dictionary: The class dictionary of the class we're - constructing. dictionary[_DESCRIPTOR_KEY] must contain - a Descriptor object describing this protocol message - type. - """ - descriptor = dictionary[GeneratedProtocolMessageType._DESCRIPTOR_KEY] - _InitMessage(descriptor, cls) - superclass = super(GeneratedProtocolMessageType, cls) - superclass.__init__(name, bases, dictionary) - - -def ParseMessage(descriptor, byte_str): - """Generate a new Message instance from this Descriptor and a byte string. - - Args: - descriptor: Protobuf Descriptor object - byte_str: Serialized protocol buffer byte string - - Returns: - Newly created protobuf Message object. - """ - - class _ResultClass(message.Message): - __metaclass__ = GeneratedProtocolMessageType - DESCRIPTOR = descriptor - - new_msg = _ResultClass() - new_msg.ParseFromString(byte_str) - return new_msg diff --git a/generator/google/protobuf/service.py b/generator/google/protobuf/service.py deleted file mode 100644 index 180b70e..0000000 --- a/generator/google/protobuf/service.py +++ /dev/null @@ -1,226 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""DEPRECATED: Declares the RPC service interfaces. - -This module declares the abstract interfaces underlying proto2 RPC -services. These are intended to be independent of any particular RPC -implementation, so that proto2 services can be used on top of a variety -of implementations. Starting with version 2.3.0, RPC implementations should -not try to build on these, but should instead provide code generator plugins -which generate code specific to the particular RPC implementation. This way -the generated code can be more appropriate for the implementation in use -and can avoid unnecessary layers of indirection. -""" - -__author__ = 'petar@google.com (Petar Petrov)' - - -class RpcException(Exception): - """Exception raised on failed blocking RPC method call.""" - pass - - -class Service(object): - - """Abstract base interface for protocol-buffer-based RPC services. - - Services themselves are abstract classes (implemented either by servers or as - stubs), but they subclass this base interface. The methods of this - interface can be used to call the methods of the service without knowing - its exact type at compile time (analogous to the Message interface). - """ - - def GetDescriptor(): - """Retrieves this service's descriptor.""" - raise NotImplementedError - - def CallMethod(self, method_descriptor, rpc_controller, - request, done): - """Calls a method of the service specified by method_descriptor. - - If "done" is None then the call is blocking and the response - message will be returned directly. Otherwise the call is asynchronous - and "done" will later be called with the response value. - - In the blocking case, RpcException will be raised on error. - - Preconditions: - * method_descriptor.service == GetDescriptor - * request is of the exact same classes as returned by - GetRequestClass(method). - * After the call has started, the request must not be modified. - * "rpc_controller" is of the correct type for the RPC implementation being - used by this Service. For stubs, the "correct type" depends on the - RpcChannel which the stub is using. - - Postconditions: - * "done" will be called when the method is complete. This may be - before CallMethod() returns or it may be at some point in the future. - * If the RPC failed, the response value passed to "done" will be None. - Further details about the failure can be found by querying the - RpcController. - """ - raise NotImplementedError - - def GetRequestClass(self, method_descriptor): - """Returns the class of the request message for the specified method. - - CallMethod() requires that the request is of a particular subclass of - Message. GetRequestClass() gets the default instance of this required - type. - - Example: - method = service.GetDescriptor().FindMethodByName("Foo") - request = stub.GetRequestClass(method)() - request.ParseFromString(input) - service.CallMethod(method, request, callback) - """ - raise NotImplementedError - - def GetResponseClass(self, method_descriptor): - """Returns the class of the response message for the specified method. - - This method isn't really needed, as the RpcChannel's CallMethod constructs - the response protocol message. It's provided anyway in case it is useful - for the caller to know the response type in advance. - """ - raise NotImplementedError - - -class RpcController(object): - - """An RpcController mediates a single method call. - - The primary purpose of the controller is to provide a way to manipulate - settings specific to the RPC implementation and to find out about RPC-level - errors. The methods provided by the RpcController interface are intended - to be a "least common denominator" set of features which we expect all - implementations to support. Specific implementations may provide more - advanced features (e.g. deadline propagation). - """ - - # Client-side methods below - - def Reset(self): - """Resets the RpcController to its initial state. - - After the RpcController has been reset, it may be reused in - a new call. Must not be called while an RPC is in progress. - """ - raise NotImplementedError - - def Failed(self): - """Returns true if the call failed. - - After a call has finished, returns true if the call failed. The possible - reasons for failure depend on the RPC implementation. Failed() must not - be called before a call has finished. If Failed() returns true, the - contents of the response message are undefined. - """ - raise NotImplementedError - - def ErrorText(self): - """If Failed is true, returns a human-readable description of the error.""" - raise NotImplementedError - - def StartCancel(self): - """Initiate cancellation. - - Advises the RPC system that the caller desires that the RPC call be - canceled. The RPC system may cancel it immediately, may wait awhile and - then cancel it, or may not even cancel the call at all. If the call is - canceled, the "done" callback will still be called and the RpcController - will indicate that the call failed at that time. - """ - raise NotImplementedError - - # Server-side methods below - - def SetFailed(self, reason): - """Sets a failure reason. - - Causes Failed() to return true on the client side. "reason" will be - incorporated into the message returned by ErrorText(). If you find - you need to return machine-readable information about failures, you - should incorporate it into your response protocol buffer and should - NOT call SetFailed(). - """ - raise NotImplementedError - - def IsCanceled(self): - """Checks if the client cancelled the RPC. - - If true, indicates that the client canceled the RPC, so the server may - as well give up on replying to it. The server should still call the - final "done" callback. - """ - raise NotImplementedError - - def NotifyOnCancel(self, callback): - """Sets a callback to invoke on cancel. - - Asks that the given callback be called when the RPC is canceled. The - callback will always be called exactly once. If the RPC completes without - being canceled, the callback will be called after completion. If the RPC - has already been canceled when NotifyOnCancel() is called, the callback - will be called immediately. - - NotifyOnCancel() must be called no more than once per request. - """ - raise NotImplementedError - - -class RpcChannel(object): - - """Abstract interface for an RPC channel. - - An RpcChannel represents a communication line to a service which can be used - to call that service's methods. The service may be running on another - machine. Normally, you should not use an RpcChannel directly, but instead - construct a stub {@link Service} wrapping it. Example: - - Example: - RpcChannel channel = rpcImpl.Channel("remotehost.example.com:1234") - RpcController controller = rpcImpl.Controller() - MyService service = MyService_Stub(channel) - service.MyMethod(controller, request, callback) - """ - - def CallMethod(self, method_descriptor, rpc_controller, - request, response_class, done): - """Calls the method identified by the descriptor. - - Call the given method of the remote service. The signature of this - procedure looks the same as Service.CallMethod(), but the requirements - are less strict in one important way: the request object doesn't have to - be of any specific class as long as its descriptor is method.input_type. - """ - raise NotImplementedError diff --git a/generator/google/protobuf/service_reflection.py b/generator/google/protobuf/service_reflection.py deleted file mode 100644 index 851e83e..0000000 --- a/generator/google/protobuf/service_reflection.py +++ /dev/null @@ -1,284 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Contains metaclasses used to create protocol service and service stub -classes from ServiceDescriptor objects at runtime. - -The GeneratedServiceType and GeneratedServiceStubType metaclasses are used to -inject all useful functionality into the classes output by the protocol -compiler at compile-time. -""" - -__author__ = 'petar@google.com (Petar Petrov)' - - -class GeneratedServiceType(type): - - """Metaclass for service classes created at runtime from ServiceDescriptors. - - Implementations for all methods described in the Service class are added here - by this class. We also create properties to allow getting/setting all fields - in the protocol message. - - The protocol compiler currently uses this metaclass to create protocol service - classes at runtime. Clients can also manually create their own classes at - runtime, as in this example: - - mydescriptor = ServiceDescriptor(.....) - class MyProtoService(service.Service): - __metaclass__ = GeneratedServiceType - DESCRIPTOR = mydescriptor - myservice_instance = MyProtoService() - ... - """ - - _DESCRIPTOR_KEY = 'DESCRIPTOR' - - def __init__(cls, name, bases, dictionary): - """Creates a message service class. - - Args: - name: Name of the class (ignored, but required by the metaclass - protocol). - bases: Base classes of the class being constructed. - dictionary: The class dictionary of the class being constructed. - dictionary[_DESCRIPTOR_KEY] must contain a ServiceDescriptor object - describing this protocol service type. - """ - # Don't do anything if this class doesn't have a descriptor. This happens - # when a service class is subclassed. - if GeneratedServiceType._DESCRIPTOR_KEY not in dictionary: - return - descriptor = dictionary[GeneratedServiceType._DESCRIPTOR_KEY] - service_builder = _ServiceBuilder(descriptor) - service_builder.BuildService(cls) - - -class GeneratedServiceStubType(GeneratedServiceType): - - """Metaclass for service stubs created at runtime from ServiceDescriptors. - - This class has similar responsibilities as GeneratedServiceType, except that - it creates the service stub classes. - """ - - _DESCRIPTOR_KEY = 'DESCRIPTOR' - - def __init__(cls, name, bases, dictionary): - """Creates a message service stub class. - - Args: - name: Name of the class (ignored, here). - bases: Base classes of the class being constructed. - dictionary: The class dictionary of the class being constructed. - dictionary[_DESCRIPTOR_KEY] must contain a ServiceDescriptor object - describing this protocol service type. - """ - super(GeneratedServiceStubType, cls).__init__(name, bases, dictionary) - # Don't do anything if this class doesn't have a descriptor. This happens - # when a service stub is subclassed. - if GeneratedServiceStubType._DESCRIPTOR_KEY not in dictionary: - return - descriptor = dictionary[GeneratedServiceStubType._DESCRIPTOR_KEY] - service_stub_builder = _ServiceStubBuilder(descriptor) - service_stub_builder.BuildServiceStub(cls) - - -class _ServiceBuilder(object): - - """This class constructs a protocol service class using a service descriptor. - - Given a service descriptor, this class constructs a class that represents - the specified service descriptor. One service builder instance constructs - exactly one service class. That means all instances of that class share the - same builder. - """ - - def __init__(self, service_descriptor): - """Initializes an instance of the service class builder. - - Args: - service_descriptor: ServiceDescriptor to use when constructing the - service class. - """ - self.descriptor = service_descriptor - - def BuildService(self, cls): - """Constructs the service class. - - Args: - cls: The class that will be constructed. - """ - - # CallMethod needs to operate with an instance of the Service class. This - # internal wrapper function exists only to be able to pass the service - # instance to the method that does the real CallMethod work. - def _WrapCallMethod(srvc, method_descriptor, - rpc_controller, request, callback): - return self._CallMethod(srvc, method_descriptor, - rpc_controller, request, callback) - self.cls = cls - cls.CallMethod = _WrapCallMethod - cls.GetDescriptor = staticmethod(lambda: self.descriptor) - cls.GetDescriptor.__doc__ = "Returns the service descriptor." - cls.GetRequestClass = self._GetRequestClass - cls.GetResponseClass = self._GetResponseClass - for method in self.descriptor.methods: - setattr(cls, method.name, self._GenerateNonImplementedMethod(method)) - - def _CallMethod(self, srvc, method_descriptor, - rpc_controller, request, callback): - """Calls the method described by a given method descriptor. - - Args: - srvc: Instance of the service for which this method is called. - method_descriptor: Descriptor that represent the method to call. - rpc_controller: RPC controller to use for this method's execution. - request: Request protocol message. - callback: A callback to invoke after the method has completed. - """ - if method_descriptor.containing_service != self.descriptor: - raise RuntimeError( - 'CallMethod() given method descriptor for wrong service type.') - method = getattr(srvc, method_descriptor.name) - return method(rpc_controller, request, callback) - - def _GetRequestClass(self, method_descriptor): - """Returns the class of the request protocol message. - - Args: - method_descriptor: Descriptor of the method for which to return the - request protocol message class. - - Returns: - A class that represents the input protocol message of the specified - method. - """ - if method_descriptor.containing_service != self.descriptor: - raise RuntimeError( - 'GetRequestClass() given method descriptor for wrong service type.') - return method_descriptor.input_type._concrete_class - - def _GetResponseClass(self, method_descriptor): - """Returns the class of the response protocol message. - - Args: - method_descriptor: Descriptor of the method for which to return the - response protocol message class. - - Returns: - A class that represents the output protocol message of the specified - method. - """ - if method_descriptor.containing_service != self.descriptor: - raise RuntimeError( - 'GetResponseClass() given method descriptor for wrong service type.') - return method_descriptor.output_type._concrete_class - - def _GenerateNonImplementedMethod(self, method): - """Generates and returns a method that can be set for a service methods. - - Args: - method: Descriptor of the service method for which a method is to be - generated. - - Returns: - A method that can be added to the service class. - """ - return lambda inst, rpc_controller, request, callback: ( - self._NonImplementedMethod(method.name, rpc_controller, callback)) - - def _NonImplementedMethod(self, method_name, rpc_controller, callback): - """The body of all methods in the generated service class. - - Args: - method_name: Name of the method being executed. - rpc_controller: RPC controller used to execute this method. - callback: A callback which will be invoked when the method finishes. - """ - rpc_controller.SetFailed('Method %s not implemented.' % method_name) - callback(None) - - -class _ServiceStubBuilder(object): - - """Constructs a protocol service stub class using a service descriptor. - - Given a service descriptor, this class constructs a suitable stub class. - A stub is just a type-safe wrapper around an RpcChannel which emulates a - local implementation of the service. - - One service stub builder instance constructs exactly one class. It means all - instances of that class share the same service stub builder. - """ - - def __init__(self, service_descriptor): - """Initializes an instance of the service stub class builder. - - Args: - service_descriptor: ServiceDescriptor to use when constructing the - stub class. - """ - self.descriptor = service_descriptor - - def BuildServiceStub(self, cls): - """Constructs the stub class. - - Args: - cls: The class that will be constructed. - """ - - def _ServiceStubInit(stub, rpc_channel): - stub.rpc_channel = rpc_channel - self.cls = cls - cls.__init__ = _ServiceStubInit - for method in self.descriptor.methods: - setattr(cls, method.name, self._GenerateStubMethod(method)) - - def _GenerateStubMethod(self, method): - return (lambda inst, rpc_controller, request, callback=None: - self._StubMethod(inst, method, rpc_controller, request, callback)) - - def _StubMethod(self, stub, method_descriptor, - rpc_controller, request, callback): - """The body of all service methods in the generated stub class. - - Args: - stub: Stub instance. - method_descriptor: Descriptor of the invoked method. - rpc_controller: Rpc controller to execute the method. - request: Request protocol message. - callback: A callback to execute when the method finishes. - Returns: - Response message (in case of blocking call). - """ - return stub.rpc_channel.CallMethod( - method_descriptor, rpc_controller, request, - method_descriptor.output_type._concrete_class, callback) diff --git a/generator/google/protobuf/text_format.py b/generator/google/protobuf/text_format.py deleted file mode 100644 index 24dd07f..0000000 --- a/generator/google/protobuf/text_format.py +++ /dev/null @@ -1,739 +0,0 @@ -# Protocol Buffers - Google's data interchange format -# Copyright 2008 Google Inc. All rights reserved. -# http://code.google.com/p/protobuf/ -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are -# met: -# -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following disclaimer -# in the documentation and/or other materials provided with the -# distribution. -# * Neither the name of Google Inc. nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -"""Contains routines for printing protocol messages in text format.""" - -__author__ = 'kenton@google.com (Kenton Varda)' - -import cStringIO -import re - -from collections import deque -from google.protobuf.internal import type_checkers -from google.protobuf import descriptor - -__all__ = [ 'MessageToString', 'PrintMessage', 'PrintField', - 'PrintFieldValue', 'Merge' ] - - -_INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(), - type_checkers.Int32ValueChecker(), - type_checkers.Uint64ValueChecker(), - type_checkers.Int64ValueChecker()) -_FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?', re.IGNORECASE) -_FLOAT_NAN = re.compile('nanf?', re.IGNORECASE) - - -class ParseError(Exception): - """Thrown in case of ASCII parsing error.""" - - -def MessageToString(message, as_utf8=False, as_one_line=False): - out = cStringIO.StringIO() - PrintMessage(message, out, as_utf8=as_utf8, as_one_line=as_one_line) - result = out.getvalue() - out.close() - if as_one_line: - return result.rstrip() - return result - - -def PrintMessage(message, out, indent=0, as_utf8=False, as_one_line=False): - for field, value in message.ListFields(): - if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: - for element in value: - PrintField(field, element, out, indent, as_utf8, as_one_line) - else: - PrintField(field, value, out, indent, as_utf8, as_one_line) - - -def PrintField(field, value, out, indent=0, as_utf8=False, as_one_line=False): - """Print a single field name/value pair. For repeated fields, the value - should be a single element.""" - - out.write(' ' * indent); - if field.is_extension: - out.write('[') - if (field.containing_type.GetOptions().message_set_wire_format and - field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and - field.message_type == field.extension_scope and - field.label == descriptor.FieldDescriptor.LABEL_OPTIONAL): - out.write(field.message_type.full_name) - else: - out.write(field.full_name) - out.write(']') - elif field.type == descriptor.FieldDescriptor.TYPE_GROUP: - # For groups, use the capitalized name. - out.write(field.message_type.name) - else: - out.write(field.name) - - if field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE: - # The colon is optional in this case, but our cross-language golden files - # don't include it. - out.write(': ') - - PrintFieldValue(field, value, out, indent, as_utf8, as_one_line) - if as_one_line: - out.write(' ') - else: - out.write('\n') - - -def PrintFieldValue(field, value, out, indent=0, - as_utf8=False, as_one_line=False): - """Print a single field value (not including name). For repeated fields, - the value should be a single element.""" - - if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: - if as_one_line: - out.write(' { ') - PrintMessage(value, out, indent, as_utf8, as_one_line) - out.write('}') - else: - out.write(' {\n') - PrintMessage(value, out, indent + 2, as_utf8, as_one_line) - out.write(' ' * indent + '}') - elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: - enum_value = field.enum_type.values_by_number.get(value, None) - if enum_value is not None: - out.write(enum_value.name) - else: - out.write(str(value)) - elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: - out.write('\"') - if type(value) is unicode: - out.write(_CEscape(value.encode('utf-8'), as_utf8)) - else: - out.write(_CEscape(value, as_utf8)) - out.write('\"') - elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: - if value: - out.write("true") - else: - out.write("false") - else: - out.write(str(value)) - - -def Merge(text, message): - """Merges an ASCII representation of a protocol message into a message. - - Args: - text: Message ASCII representation. - message: A protocol buffer message to merge into. - - Raises: - ParseError: On ASCII parsing problems. - """ - tokenizer = _Tokenizer(text) - while not tokenizer.AtEnd(): - _MergeField(tokenizer, message) - - -def _MergeField(tokenizer, message): - """Merges a single protocol message field into a message. - - Args: - tokenizer: A tokenizer to parse the field name and values. - message: A protocol message to record the data. - - Raises: - ParseError: In case of ASCII parsing problems. - """ - message_descriptor = message.DESCRIPTOR - if tokenizer.TryConsume('['): - name = [tokenizer.ConsumeIdentifier()] - while tokenizer.TryConsume('.'): - name.append(tokenizer.ConsumeIdentifier()) - name = '.'.join(name) - - if not message_descriptor.is_extendable: - raise tokenizer.ParseErrorPreviousToken( - 'Message type "%s" does not have extensions.' % - message_descriptor.full_name) - field = message.Extensions._FindExtensionByName(name) - if not field: - raise tokenizer.ParseErrorPreviousToken( - 'Extension "%s" not registered.' % name) - elif message_descriptor != field.containing_type: - raise tokenizer.ParseErrorPreviousToken( - 'Extension "%s" does not extend message type "%s".' % ( - name, message_descriptor.full_name)) - tokenizer.Consume(']') - else: - name = tokenizer.ConsumeIdentifier() - field = message_descriptor.fields_by_name.get(name, None) - - # Group names are expected to be capitalized as they appear in the - # .proto file, which actually matches their type names, not their field - # names. - if not field: - field = message_descriptor.fields_by_name.get(name.lower(), None) - if field and field.type != descriptor.FieldDescriptor.TYPE_GROUP: - field = None - - if (field and field.type == descriptor.FieldDescriptor.TYPE_GROUP and - field.message_type.name != name): - field = None - - if not field: - raise tokenizer.ParseErrorPreviousToken( - 'Message type "%s" has no field named "%s".' % ( - message_descriptor.full_name, name)) - - if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: - tokenizer.TryConsume(':') - - if tokenizer.TryConsume('<'): - end_token = '>' - else: - tokenizer.Consume('{') - end_token = '}' - - if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: - if field.is_extension: - sub_message = message.Extensions[field].add() - else: - sub_message = getattr(message, field.name).add() - else: - if field.is_extension: - sub_message = message.Extensions[field] - else: - sub_message = getattr(message, field.name) - sub_message.SetInParent() - - while not tokenizer.TryConsume(end_token): - if tokenizer.AtEnd(): - raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token)) - _MergeField(tokenizer, sub_message) - else: - _MergeScalarField(tokenizer, message, field) - - -def _MergeScalarField(tokenizer, message, field): - """Merges a single protocol message scalar field into a message. - - Args: - tokenizer: A tokenizer to parse the field value. - message: A protocol message to record the data. - field: The descriptor of the field to be merged. - - Raises: - ParseError: In case of ASCII parsing problems. - RuntimeError: On runtime errors. - """ - tokenizer.Consume(':') - value = None - - if field.type in (descriptor.FieldDescriptor.TYPE_INT32, - descriptor.FieldDescriptor.TYPE_SINT32, - descriptor.FieldDescriptor.TYPE_SFIXED32): - value = tokenizer.ConsumeInt32() - elif field.type in (descriptor.FieldDescriptor.TYPE_INT64, - descriptor.FieldDescriptor.TYPE_SINT64, - descriptor.FieldDescriptor.TYPE_SFIXED64): - value = tokenizer.ConsumeInt64() - elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32, - descriptor.FieldDescriptor.TYPE_FIXED32): - value = tokenizer.ConsumeUint32() - elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64, - descriptor.FieldDescriptor.TYPE_FIXED64): - value = tokenizer.ConsumeUint64() - elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT, - descriptor.FieldDescriptor.TYPE_DOUBLE): - value = tokenizer.ConsumeFloat() - elif field.type == descriptor.FieldDescriptor.TYPE_BOOL: - value = tokenizer.ConsumeBool() - elif field.type == descriptor.FieldDescriptor.TYPE_STRING: - value = tokenizer.ConsumeString() - elif field.type == descriptor.FieldDescriptor.TYPE_BYTES: - value = tokenizer.ConsumeByteString() - elif field.type == descriptor.FieldDescriptor.TYPE_ENUM: - value = tokenizer.ConsumeEnum(field) - else: - raise RuntimeError('Unknown field type %d' % field.type) - - if field.label == descriptor.FieldDescriptor.LABEL_REPEATED: - if field.is_extension: - message.Extensions[field].append(value) - else: - getattr(message, field.name).append(value) - else: - if field.is_extension: - message.Extensions[field] = value - else: - setattr(message, field.name, value) - - -class _Tokenizer(object): - """Protocol buffer ASCII representation tokenizer. - - This class handles the lower level string parsing by splitting it into - meaningful tokens. - - It was directly ported from the Java protocol buffer API. - """ - - _WHITESPACE = re.compile('(\\s|(#.*$))+', re.MULTILINE) - _TOKEN = re.compile( - '[a-zA-Z_][0-9a-zA-Z_+-]*|' # an identifier - '[0-9+-][0-9a-zA-Z_.+-]*|' # a number - '\"([^\"\n\\\\]|\\\\.)*(\"|\\\\?$)|' # a double-quoted string - '\'([^\'\n\\\\]|\\\\.)*(\'|\\\\?$)') # a single-quoted string - _IDENTIFIER = re.compile('\w+') - - def __init__(self, text_message): - self._text_message = text_message - - self._position = 0 - self._line = -1 - self._column = 0 - self._token_start = None - self.token = '' - self._lines = deque(text_message.split('\n')) - self._current_line = '' - self._previous_line = 0 - self._previous_column = 0 - self._SkipWhitespace() - self.NextToken() - - def AtEnd(self): - """Checks the end of the text was reached. - - Returns: - True iff the end was reached. - """ - return self.token == '' - - def _PopLine(self): - while len(self._current_line) <= self._column: - if not self._lines: - self._current_line = '' - return - self._line += 1 - self._column = 0 - self._current_line = self._lines.popleft() - - def _SkipWhitespace(self): - while True: - self._PopLine() - match = self._WHITESPACE.match(self._current_line, self._column) - if not match: - break - length = len(match.group(0)) - self._column += length - - def TryConsume(self, token): - """Tries to consume a given piece of text. - - Args: - token: Text to consume. - - Returns: - True iff the text was consumed. - """ - if self.token == token: - self.NextToken() - return True - return False - - def Consume(self, token): - """Consumes a piece of text. - - Args: - token: Text to consume. - - Raises: - ParseError: If the text couldn't be consumed. - """ - if not self.TryConsume(token): - raise self._ParseError('Expected "%s".' % token) - - def ConsumeIdentifier(self): - """Consumes protocol message field identifier. - - Returns: - Identifier string. - - Raises: - ParseError: If an identifier couldn't be consumed. - """ - result = self.token - if not self._IDENTIFIER.match(result): - raise self._ParseError('Expected identifier.') - self.NextToken() - return result - - def ConsumeInt32(self): - """Consumes a signed 32bit integer number. - - Returns: - The integer parsed. - - Raises: - ParseError: If a signed 32bit integer couldn't be consumed. - """ - try: - result = ParseInteger(self.token, is_signed=True, is_long=False) - except ValueError, e: - raise self._ParseError(str(e)) - self.NextToken() - return result - - def ConsumeUint32(self): - """Consumes an unsigned 32bit integer number. - - Returns: - The integer parsed. - - Raises: - ParseError: If an unsigned 32bit integer couldn't be consumed. - """ - try: - result = ParseInteger(self.token, is_signed=False, is_long=False) - except ValueError, e: - raise self._ParseError(str(e)) - self.NextToken() - return result - - def ConsumeInt64(self): - """Consumes a signed 64bit integer number. - - Returns: - The integer parsed. - - Raises: - ParseError: If a signed 64bit integer couldn't be consumed. - """ - try: - result = ParseInteger(self.token, is_signed=True, is_long=True) - except ValueError, e: - raise self._ParseError(str(e)) - self.NextToken() - return result - - def ConsumeUint64(self): - """Consumes an unsigned 64bit integer number. - - Returns: - The integer parsed. - - Raises: - ParseError: If an unsigned 64bit integer couldn't be consumed. - """ - try: - result = ParseInteger(self.token, is_signed=False, is_long=True) - except ValueError, e: - raise self._ParseError(str(e)) - self.NextToken() - return result - - def ConsumeFloat(self): - """Consumes an floating point number. - - Returns: - The number parsed. - - Raises: - ParseError: If a floating point number couldn't be consumed. - """ - try: - result = ParseFloat(self.token) - except ValueError, e: - raise self._ParseError(str(e)) - self.NextToken() - return result - - def ConsumeBool(self): - """Consumes a boolean value. - - Returns: - The bool parsed. - - Raises: - ParseError: If a boolean value couldn't be consumed. - """ - try: - result = ParseBool(self.token) - except ValueError, e: - raise self._ParseError(str(e)) - self.NextToken() - return result - - def ConsumeString(self): - """Consumes a string value. - - Returns: - The string parsed. - - Raises: - ParseError: If a string value couldn't be consumed. - """ - bytes = self.ConsumeByteString() - try: - return unicode(bytes, 'utf-8') - except UnicodeDecodeError, e: - raise self._StringParseError(e) - - def ConsumeByteString(self): - """Consumes a byte array value. - - Returns: - The array parsed (as a string). - - Raises: - ParseError: If a byte array value couldn't be consumed. - """ - list = [self._ConsumeSingleByteString()] - while len(self.token) > 0 and self.token[0] in ('\'', '"'): - list.append(self._ConsumeSingleByteString()) - return "".join(list) - - def _ConsumeSingleByteString(self): - """Consume one token of a string literal. - - String literals (whether bytes or text) can come in multiple adjacent - tokens which are automatically concatenated, like in C or Python. This - method only consumes one token. - """ - text = self.token - if len(text) < 1 or text[0] not in ('\'', '"'): - raise self._ParseError('Expected string.') - - if len(text) < 2 or text[-1] != text[0]: - raise self._ParseError('String missing ending quote.') - - try: - result = _CUnescape(text[1:-1]) - except ValueError, e: - raise self._ParseError(str(e)) - self.NextToken() - return result - - def ConsumeEnum(self, field): - try: - result = ParseEnum(field, self.token) - except ValueError, e: - raise self._ParseError(str(e)) - self.NextToken() - return result - - def ParseErrorPreviousToken(self, message): - """Creates and *returns* a ParseError for the previously read token. - - Args: - message: A message to set for the exception. - - Returns: - A ParseError instance. - """ - return ParseError('%d:%d : %s' % ( - self._previous_line + 1, self._previous_column + 1, message)) - - def _ParseError(self, message): - """Creates and *returns* a ParseError for the current token.""" - return ParseError('%d:%d : %s' % ( - self._line + 1, self._column + 1, message)) - - def _StringParseError(self, e): - return self._ParseError('Couldn\'t parse string: ' + str(e)) - - def NextToken(self): - """Reads the next meaningful token.""" - self._previous_line = self._line - self._previous_column = self._column - - self._column += len(self.token) - self._SkipWhitespace() - - if not self._lines and len(self._current_line) <= self._column: - self.token = '' - return - - match = self._TOKEN.match(self._current_line, self._column) - if match: - token = match.group(0) - self.token = token - else: - self.token = self._current_line[self._column] - - -# text.encode('string_escape') does not seem to satisfy our needs as it -# encodes unprintable characters using two-digit hex escapes whereas our -# C++ unescaping function allows hex escapes to be any length. So, -# "\0011".encode('string_escape') ends up being "\\x011", which will be -# decoded in C++ as a single-character string with char code 0x11. -def _CEscape(text, as_utf8): - def escape(c): - o = ord(c) - if o == 10: return r"\n" # optional escape - if o == 13: return r"\r" # optional escape - if o == 9: return r"\t" # optional escape - if o == 39: return r"\'" # optional escape - - if o == 34: return r'\"' # necessary escape - if o == 92: return r"\\" # necessary escape - - # necessary escapes - if not as_utf8 and (o >= 127 or o < 32): return "\\%03o" % o - return c - return "".join([escape(c) for c in text]) - - -_CUNESCAPE_HEX = re.compile(r'(\\+)x([0-9a-fA-F])(?![0-9a-fA-F])') - - -def _CUnescape(text): - def ReplaceHex(m): - # Only replace the match if the number of leading back slashes is odd. i.e. - # the slash itself is not escaped. - if len(m.group(1)) & 1: - return m.group(1) + 'x0' + m.group(2) - return m.group(0) - - # This is required because the 'string_escape' encoding doesn't - # allow single-digit hex escapes (like '\xf'). - result = _CUNESCAPE_HEX.sub(ReplaceHex, text) - return result.decode('string_escape') - - -def ParseInteger(text, is_signed=False, is_long=False): - """Parses an integer. - - Args: - text: The text to parse. - is_signed: True if a signed integer must be parsed. - is_long: True if a long integer must be parsed. - - Returns: - The integer value. - - Raises: - ValueError: Thrown Iff the text is not a valid integer. - """ - # Do the actual parsing. Exception handling is propagated to caller. - try: - result = int(text, 0) - except ValueError: - raise ValueError('Couldn\'t parse integer: %s' % text) - - # Check if the integer is sane. Exceptions handled by callers. - checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)] - checker.CheckValue(result) - return result - - -def ParseFloat(text): - """Parse a floating point number. - - Args: - text: Text to parse. - - Returns: - The number parsed. - - Raises: - ValueError: If a floating point number couldn't be parsed. - """ - try: - # Assume Python compatible syntax. - return float(text) - except ValueError: - # Check alternative spellings. - if _FLOAT_INFINITY.match(text): - if text[0] == '-': - return float('-inf') - else: - return float('inf') - elif _FLOAT_NAN.match(text): - return float('nan') - else: - # assume '1.0f' format - try: - return float(text.rstrip('f')) - except ValueError: - raise ValueError('Couldn\'t parse float: %s' % text) - - -def ParseBool(text): - """Parse a boolean value. - - Args: - text: Text to parse. - - Returns: - Boolean values parsed - - Raises: - ValueError: If text is not a valid boolean. - """ - if text in ('true', 't', '1'): - return True - elif text in ('false', 'f', '0'): - return False - else: - raise ValueError('Expected "true" or "false".') - - -def ParseEnum(field, value): - """Parse an enum value. - - The value can be specified by a number (the enum value), or by - a string literal (the enum name). - - Args: - field: Enum field descriptor. - value: String value. - - Returns: - Enum value number. - - Raises: - ValueError: If the enum value could not be parsed. - """ - enum_descriptor = field.enum_type - try: - number = int(value, 0) - except ValueError: - # Identifier. - enum_value = enum_descriptor.values_by_name.get(value, None) - if enum_value is None: - raise ValueError( - 'Enum type "%s" has no value named %s.' % ( - enum_descriptor.full_name, value)) - else: - # Numeric value. - enum_value = enum_descriptor.values_by_number.get(number, None) - if enum_value is None: - raise ValueError( - 'Enum type "%s" has no value with number %d.' % ( - enum_descriptor.full_name, number)) - return enum_value.number diff --git a/generator/nanopb/options.proto b/generator/nanopb/options.proto new file mode 100644 index 0000000..b94dca2 --- /dev/null +++ b/generator/nanopb/options.proto @@ -0,0 +1,123 @@ +// This is a transitional file, to provide parallel support between the old +// nanopb.proto and new options.proto files. Eventually nanopb.proto will +// be left only for legacy code, but for now the generator is still also +// using it. However, your new code can start using this file already now. +// See pull request #241 for details: +// https://github.com/nanopb/nanopb/pull/241 + +// Custom options for defining: +// - Maximum size of string/bytes +// - Maximum number of elements in array +// +// These are used by nanopb to generate statically allocable structures +// for memory-limited environments. + +syntax = "proto2"; +import "google/protobuf/descriptor.proto"; + +package nanopb; +option java_package = "fi.kapsi.koti.jpa.nanopb"; + +enum FieldType { + FT_DEFAULT = 0; // Automatically decide field type, generate static field if possible. + FT_CALLBACK = 1; // Always generate a callback field. + FT_POINTER = 4; // Always generate a dynamically allocated field. + FT_STATIC = 2; // Generate a static field or raise an exception if not possible. + FT_IGNORE = 3; // Ignore the field completely. + FT_INLINE = 5; // Legacy option, use the separate 'fixed_length' option instead +} + +enum IntSize { + IS_DEFAULT = 0; // Default, 32/64bit based on type in .proto + IS_8 = 8; + IS_16 = 16; + IS_32 = 32; + IS_64 = 64; +} + +// This is the inner options message, which basically defines options for +// a field. When it is used in message or file scope, it applies to all +// fields. +message Options { + // Allocated size for 'bytes' and 'string' fields. + // For string fields, this should include the space for null terminator. + optional int32 max_size = 1; + + // Maximum length for 'string' fields. Setting this is equivalent + // to setting max_size to a value of length+1. + optional int32 max_length = 14; + + // Allocated number of entries in arrays ('repeated' fields) + optional int32 max_count = 2; + + // Size of integer fields. Can save some memory if you don't need + // full 32 bits for the value. + optional IntSize int_size = 7 [default = IS_DEFAULT]; + + // Force type of field (callback or static allocation) + optional FieldType type = 3 [default = FT_DEFAULT]; + + // Use long names for enums, i.e. EnumName_EnumValue. + optional bool long_names = 4 [default = true]; + + // Add 'packed' attribute to generated structs. + // Note: this cannot be used on CPUs that break on unaligned + // accesses to variables. + optional bool packed_struct = 5 [default = false]; + + // Add 'packed' attribute to generated enums. + optional bool packed_enum = 10 [default = false]; + + // Skip this message + optional bool skip_message = 6 [default = false]; + + // Generate oneof fields as normal optional fields instead of union. + optional bool no_unions = 8 [default = false]; + + // integer type tag for a message + optional uint32 msgid = 9; + + // decode oneof as anonymous union + optional bool anonymous_oneof = 11 [default = false]; + + // Proto3 singular field does not generate a "has_" flag + optional bool proto3 = 12 [default = false]; + + // Generate an enum->string mapping function (can take up lots of space). + optional bool enum_to_string = 13 [default = false]; + + // Generate bytes arrays with fixed length + optional bool fixed_length = 15 [default = false]; + + // Generate repeated field with fixed count + optional bool fixed_count = 16 [default = false]; +} + +// Extensions to protoc 'Descriptor' type in order to define options +// inside a .proto file. +// +// Protocol Buffers extension number registry +// -------------------------------- +// Project: Nanopb +// Contact: Petteri Aimonen <jpa@kapsi.fi> +// Web site: http://kapsi.fi/~jpa/nanopb +// Extensions: 1010 (all types) +// -------------------------------- + +extend google.protobuf.FileOptions { + optional Options fileopt = 1010; +} + +extend google.protobuf.MessageOptions { + optional Options msgopt = 1010; +} + +extend google.protobuf.EnumOptions { + optional Options enumopt = 1010; +} + +extend google.protobuf.FieldOptions { + optional Options fieldopt = 1010; +} + + diff --git a/generator/nanopb_generator.py b/generator/nanopb_generator.py index 4d8d733..5b74ca1 100755 --- a/generator/nanopb_generator.py +++ b/generator/nanopb_generator.py @@ -1,9 +1,14 @@ -#!/usr/bin/python +#!/usr/bin/env python + +from __future__ import unicode_literals '''Generate header file for nanopb from a ProtoBuf FileDescriptorSet.''' -nanopb_version = "nanopb-0.2.8-dev" +nanopb_version = "nanopb-0.3.9.1" import sys +import re +import codecs +from functools import reduce try: # Add some dummy imports to keep packaging tools happy. @@ -28,6 +33,20 @@ except: try: import proto.nanopb_pb2 as nanopb_pb2 import proto.plugin_pb2 as plugin_pb2 +except TypeError: + sys.stderr.write(''' + **************************************************************************** + *** Got TypeError when importing the protocol definitions for generator. *** + *** This usually means that the protoc in your path doesn't match the *** + *** Python protobuf library version. *** + *** *** + *** Please check the output of the following commands: *** + *** which protoc *** + *** protoc --version *** + *** python -c 'import google.protobuf; print(google.protobuf.__file__)' *** + **************************************************************************** + ''' + '\n') + raise except: sys.stderr.write(''' ******************************************************************** @@ -44,45 +63,59 @@ except: import time import os.path -# Values are tuple (c type, pb type, encoded size) +# Values are tuple (c type, pb type, encoded size, int_size_allowed) FieldD = descriptor.FieldDescriptorProto datatypes = { - FieldD.TYPE_BOOL: ('bool', 'BOOL', 1), - FieldD.TYPE_DOUBLE: ('double', 'DOUBLE', 8), - FieldD.TYPE_FIXED32: ('uint32_t', 'FIXED32', 4), - FieldD.TYPE_FIXED64: ('uint64_t', 'FIXED64', 8), - FieldD.TYPE_FLOAT: ('float', 'FLOAT', 4), - FieldD.TYPE_INT32: ('int32_t', 'INT32', 10), - FieldD.TYPE_INT64: ('int64_t', 'INT64', 10), - FieldD.TYPE_SFIXED32: ('int32_t', 'SFIXED32', 4), - FieldD.TYPE_SFIXED64: ('int64_t', 'SFIXED64', 8), - FieldD.TYPE_SINT32: ('int32_t', 'SINT32', 5), - FieldD.TYPE_SINT64: ('int64_t', 'SINT64', 10), - FieldD.TYPE_UINT32: ('uint32_t', 'UINT32', 5), - FieldD.TYPE_UINT64: ('uint64_t', 'UINT64', 10) + FieldD.TYPE_BOOL: ('bool', 'BOOL', 1, False), + FieldD.TYPE_DOUBLE: ('double', 'DOUBLE', 8, False), + FieldD.TYPE_FIXED32: ('uint32_t', 'FIXED32', 4, False), + FieldD.TYPE_FIXED64: ('uint64_t', 'FIXED64', 8, False), + FieldD.TYPE_FLOAT: ('float', 'FLOAT', 4, False), + FieldD.TYPE_INT32: ('int32_t', 'INT32', 10, True), + FieldD.TYPE_INT64: ('int64_t', 'INT64', 10, True), + FieldD.TYPE_SFIXED32: ('int32_t', 'SFIXED32', 4, False), + FieldD.TYPE_SFIXED64: ('int64_t', 'SFIXED64', 8, False), + FieldD.TYPE_SINT32: ('int32_t', 'SINT32', 5, True), + FieldD.TYPE_SINT64: ('int64_t', 'SINT64', 10, True), + FieldD.TYPE_UINT32: ('uint32_t', 'UINT32', 5, True), + FieldD.TYPE_UINT64: ('uint64_t', 'UINT64', 10, True) +} + +# Integer size overrides (from .proto settings) +intsizes = { + nanopb_pb2.IS_8: 'int8_t', + nanopb_pb2.IS_16: 'int16_t', + nanopb_pb2.IS_32: 'int32_t', + nanopb_pb2.IS_64: 'int64_t', } +# String types (for python 2 / python 3 compatibility) +try: + strtypes = (unicode, str) +except NameError: + strtypes = (str, ) + class Names: '''Keeps a set of nested names and formats them to C identifier.''' def __init__(self, parts = ()): if isinstance(parts, Names): parts = parts.parts self.parts = tuple(parts) - + def __str__(self): return '_'.join(self.parts) def __add__(self, other): - if isinstance(other, (str, unicode)): + if isinstance(other, strtypes): return Names(self.parts + (other,)) elif isinstance(other, tuple): return Names(self.parts + other) else: raise ValueError("Name parts should be of type str") - + def __eq__(self, other): return isinstance(other, Names) and self.parts == other.parts - + def names_from_type_name(type_name): '''Parse Names() from FieldDescriptorProto type_name''' if type_name[0] != '.': @@ -91,11 +124,14 @@ def names_from_type_name(type_name): def varint_max_size(max_value): '''Returns the maximum number of bytes a varint can take when encoded.''' + if max_value < 0: + max_value = 2**64 - max_value for i in range(1, 11): if (max_value >> (i * 7)) == 0: return i raise ValueError("Value too large for varint: " + str(max_value)) +assert varint_max_size(-1) == 10 assert varint_max_size(0) == 1 assert varint_max_size(127) == 1 assert varint_max_size(128) == 2 @@ -104,16 +140,20 @@ class EncodedSize: '''Class used to represent the encoded size of a field or a message. Consists of a combination of symbolic sizes and integer sizes.''' def __init__(self, value = 0, symbols = []): - if isinstance(value, (str, Names)): - symbols = [str(value)] - value = 0 - self.value = value - self.symbols = symbols - + if isinstance(value, EncodedSize): + self.value = value.value + self.symbols = value.symbols + elif isinstance(value, strtypes + (Names,)): + self.symbols = [str(value)] + self.value = 0 + else: + self.value = value + self.symbols = symbols + def __add__(self, other): - if isinstance(other, (int, long)): + if isinstance(other, int): return EncodedSize(self.value + other, self.symbols) - elif isinstance(other, (str, Names)): + elif isinstance(other, strtypes + (Names,)): return EncodedSize(self.value, self.symbols + [str(other)]) elif isinstance(other, EncodedSize): return EncodedSize(self.value + other.value, self.symbols + other.symbols) @@ -121,7 +161,7 @@ class EncodedSize: raise ValueError("Cannot add size: " + repr(other)) def __mul__(self, other): - if isinstance(other, (int, long)): + if isinstance(other, int): return EncodedSize(self.value * other, [str(other) + '*' + s for s in self.symbols]) else: raise ValueError("Cannot multiply size: " + repr(other)) @@ -141,28 +181,94 @@ class EncodedSize: class Enum: def __init__(self, names, desc, enum_options): '''desc is EnumDescriptorProto''' - + self.options = enum_options self.names = names + desc.name - + if enum_options.long_names: - self.values = [(self.names + x.name, x.number) for x in desc.value] + self.values = [(self.names + x.name, x.number) for x in desc.value] else: - self.values = [(names + x.name, x.number) for x in desc.value] - + self.values = [(names + x.name, x.number) for x in desc.value] + self.value_longnames = [self.names + x.name for x in desc.value] - + self.packed = enum_options.packed_enum + + def has_negative(self): + for n, v in self.values: + if v < 0: + return True + return False + + def encoded_size(self): + return max([varint_max_size(v) for n,v in self.values]) + def __str__(self): result = 'typedef enum _%s {\n' % self.names result += ',\n'.join([" %s = %d" % x for x in self.values]) - result += '\n} %s;' % self.names + result += '\n}' + + if self.packed: + result += ' pb_packed' + + result += ' %s;' % self.names + + result += '\n#define _%s_MIN %s' % (self.names, self.values[0][0]) + result += '\n#define _%s_MAX %s' % (self.names, self.values[-1][0]) + result += '\n#define _%s_ARRAYSIZE ((%s)(%s+1))' % (self.names, self.names, self.values[-1][0]) + + if not self.options.long_names: + # Define the long names always so that enum value references + # from other files work properly. + for i, x in enumerate(self.values): + result += '\n#define %s %s' % (self.value_longnames[i], x[0]) + + if self.options.enum_to_string: + result += '\nconst char *%s_name(%s v);\n' % (self.names, self.names) + return result + def enum_to_string_definition(self): + if not self.options.enum_to_string: + return "" + + result = 'const char *%s_name(%s v) {\n' % (self.names, self.names) + result += ' switch (v) {\n' + + for ((enumname, _), strname) in zip(self.values, self.value_longnames): + # Strip off the leading type name from the string value. + strval = str(strname)[len(str(self.names)) + 1:] + result += ' case %s: return "%s";\n' % (enumname, strval) + + result += ' }\n' + result += ' return "unknown";\n' + result += '}\n' + + return result + +class FieldMaxSize: + def __init__(self, worst = 0, checks = [], field_name = 'undefined'): + if isinstance(worst, list): + self.worst = max(i for i in worst if i is not None) + else: + self.worst = worst + + self.worst_field = field_name + self.checks = list(checks) + + def extend(self, extend, field_name = None): + self.worst = max(self.worst, extend.worst) + + if self.worst == extend.worst: + self.worst_field = extend.worst_field + + self.checks.extend(extend.checks) + class Field: def __init__(self, struct_name, desc, field_options): '''desc is FieldDescriptorProto''' self.tag = desc.number self.struct_name = struct_name + self.union_name = None self.name = desc.name self.default = None self.max_size = None @@ -170,51 +276,71 @@ class Field: self.array_decl = "" self.enc_size = None self.ctype = None - + self.fixed_count = False + + if field_options.type == nanopb_pb2.FT_INLINE: + # Before nanopb-0.3.8, fixed length bytes arrays were specified + # by setting type to FT_INLINE. But to handle pointer typed fields, + # it makes sense to have it as a separate option. + field_options.type = nanopb_pb2.FT_STATIC + field_options.fixed_length = True + # Parse field options if field_options.HasField("max_size"): self.max_size = field_options.max_size - + + if desc.type == FieldD.TYPE_STRING and field_options.HasField("max_length"): + # max_length overrides max_size for strings + self.max_size = field_options.max_length + 1 + if field_options.HasField("max_count"): self.max_count = field_options.max_count - + if desc.HasField('default_value'): self.default = desc.default_value - + # Check field rules, i.e. required/optional/repeated. can_be_static = True - if desc.label == FieldD.LABEL_REQUIRED: - self.rules = 'REQUIRED' - elif desc.label == FieldD.LABEL_OPTIONAL: - self.rules = 'OPTIONAL' - elif desc.label == FieldD.LABEL_REPEATED: + if desc.label == FieldD.LABEL_REPEATED: self.rules = 'REPEATED' if self.max_count is None: can_be_static = False else: self.array_decl = '[%d]' % self.max_count + self.fixed_count = field_options.fixed_count + + elif field_options.proto3: + self.rules = 'SINGULAR' + elif desc.label == FieldD.LABEL_REQUIRED: + self.rules = 'REQUIRED' + elif desc.label == FieldD.LABEL_OPTIONAL: + self.rules = 'OPTIONAL' else: raise NotImplementedError(desc.label) - + # Check if the field can be implemented with static allocation # i.e. whether the data size is known. if desc.type == FieldD.TYPE_STRING and self.max_size is None: can_be_static = False - + if desc.type == FieldD.TYPE_BYTES and self.max_size is None: can_be_static = False - + # Decide how the field data will be allocated if field_options.type == nanopb_pb2.FT_DEFAULT: if can_be_static: field_options.type = nanopb_pb2.FT_STATIC else: field_options.type = nanopb_pb2.FT_CALLBACK - + if field_options.type == nanopb_pb2.FT_STATIC and not can_be_static: - raise Exception("Field %s is defined as static, but max_size or " + raise Exception("Field '%s' is defined as static, but max_size or " "max_count is not given." % self.name) + if field_options.fixed_count and self.max_count is None: + raise Exception("Field '%s' is defined as fixed count, " + "but max_count is not given." % self.name) + if field_options.type == nanopb_pb2.FT_STATIC: self.allocation = 'STATIC' elif field_options.type == nanopb_pb2.FT_POINTER: @@ -223,16 +349,22 @@ class Field: self.allocation = 'CALLBACK' else: raise NotImplementedError(field_options.type) - + # Decide the C data type to use in the struct. - if datatypes.has_key(desc.type): - self.ctype, self.pbtype, self.enc_size = datatypes[desc.type] + if desc.type in datatypes: + self.ctype, self.pbtype, self.enc_size, isa = datatypes[desc.type] + + # Override the field size if user wants to use smaller integers + if isa and field_options.int_size != nanopb_pb2.IS_DEFAULT: + self.ctype = intsizes[field_options.int_size] + if desc.type == FieldD.TYPE_UINT32 or desc.type == FieldD.TYPE_UINT64: + self.ctype = 'u' + self.ctype; elif desc.type == FieldD.TYPE_ENUM: self.pbtype = 'ENUM' self.ctype = names_from_type_name(desc.type_name) if self.default is not None: self.default = self.ctype + self.default - self.enc_size = 5 # protoc rejects enum values > 32 bits + self.enc_size = None # Needs to be filled in when enum values are known elif desc.type == FieldD.TYPE_STRING: self.pbtype = 'STRING' self.ctype = 'char' @@ -241,31 +373,44 @@ class Field: self.array_decl += '[%d]' % self.max_size self.enc_size = varint_max_size(self.max_size) + self.max_size elif desc.type == FieldD.TYPE_BYTES: - self.pbtype = 'BYTES' - if self.allocation == 'STATIC': - self.ctype = self.struct_name + self.name + 't' + if field_options.fixed_length: + self.pbtype = 'FIXED_LENGTH_BYTES' + + if self.max_size is None: + raise Exception("Field '%s' is defined as fixed length, " + "but max_size is not given." % self.name) + self.enc_size = varint_max_size(self.max_size) + self.max_size - elif self.allocation == 'POINTER': + self.ctype = 'pb_byte_t' + self.array_decl += '[%d]' % self.max_size + else: + self.pbtype = 'BYTES' self.ctype = 'pb_bytes_array_t' + if self.allocation == 'STATIC': + self.ctype = self.struct_name + self.name + 't' + self.enc_size = varint_max_size(self.max_size) + self.max_size elif desc.type == FieldD.TYPE_MESSAGE: self.pbtype = 'MESSAGE' self.ctype = self.submsgname = names_from_type_name(desc.type_name) self.enc_size = None # Needs to be filled in after the message type is available else: raise NotImplementedError(desc.type) - - def __cmp__(self, other): - return cmp(self.tag, other.tag) - + + def __lt__(self, other): + return self.tag < other.tag + def __str__(self): result = '' if self.allocation == 'POINTER': if self.rules == 'REPEATED': - result += ' size_t ' + self.name + '_count;\n' - + result += ' pb_size_t ' + self.name + '_count;\n' + if self.pbtype == 'MESSAGE': # Use struct definition, so recursive submessages are possible result += ' struct _%s *%s;' % (self.ctype, self.name) + elif self.pbtype == 'FIXED_LENGTH_BYTES': + # Pointer to fixed size array + result += ' %s (*%s)%s;' % (self.ctype, self.name, self.array_decl) elif self.rules == 'REPEATED' and self.pbtype in ['STRING', 'BYTES']: # String/bytes arrays need to be defined as pointers to pointers result += ' %s **%s;' % (self.ctype, self.name) @@ -276,120 +421,230 @@ class Field: else: if self.rules == 'OPTIONAL' and self.allocation == 'STATIC': result += ' bool has_' + self.name + ';\n' - elif self.rules == 'REPEATED' and self.allocation == 'STATIC': - result += ' size_t ' + self.name + '_count;\n' + elif (self.rules == 'REPEATED' and + self.allocation == 'STATIC' and + not self.fixed_count): + result += ' pb_size_t ' + self.name + '_count;\n' result += ' %s %s%s;' % (self.ctype, self.name, self.array_decl) return result - + def types(self): '''Return definitions for any special types this field might need.''' if self.pbtype == 'BYTES' and self.allocation == 'STATIC': - result = 'typedef struct {\n' - result += ' size_t size;\n' - result += ' uint8_t bytes[%d];\n' % self.max_size - result += '} %s;\n' % self.ctype + result = 'typedef PB_BYTES_ARRAY_T(%d) %s;\n' % (self.max_size, self.ctype) else: - result = None + result = '' return result - + + def get_dependencies(self): + '''Get list of type names used by this field.''' + if self.allocation == 'STATIC': + return [str(self.ctype)] + else: + return [] + + def get_initializer(self, null_init, inner_init_only = False): + '''Return literal expression for this field's default value. + null_init: If True, initialize to a 0 value instead of default from .proto + inner_init_only: If True, exclude initialization for any count/has fields + ''' + + inner_init = None + if self.pbtype == 'MESSAGE': + if null_init: + inner_init = '%s_init_zero' % self.ctype + else: + inner_init = '%s_init_default' % self.ctype + elif self.default is None or null_init: + if self.pbtype == 'STRING': + inner_init = '""' + elif self.pbtype == 'BYTES': + inner_init = '{0, {0}}' + elif self.pbtype == 'FIXED_LENGTH_BYTES': + inner_init = '{0}' + elif self.pbtype in ('ENUM', 'UENUM'): + inner_init = '_%s_MIN' % self.ctype + else: + inner_init = '0' + else: + if self.pbtype == 'STRING': + data = codecs.escape_encode(self.default.encode('utf-8'))[0] + inner_init = '"' + data.decode('ascii') + '"' + elif self.pbtype == 'BYTES': + data = codecs.escape_decode(self.default)[0] + data = ["0x%02x" % c for c in bytearray(data)] + if len(data) == 0: + inner_init = '{0, {0}}' + else: + inner_init = '{%d, {%s}}' % (len(data), ','.join(data)) + elif self.pbtype == 'FIXED_LENGTH_BYTES': + data = codecs.escape_decode(self.default)[0] + data = ["0x%02x" % c for c in bytearray(data)] + if len(data) == 0: + inner_init = '{0}' + else: + inner_init = '{%s}' % ','.join(data) + elif self.pbtype in ['FIXED32', 'UINT32']: + inner_init = str(self.default) + 'u' + elif self.pbtype in ['FIXED64', 'UINT64']: + inner_init = str(self.default) + 'ull' + elif self.pbtype in ['SFIXED64', 'INT64']: + inner_init = str(self.default) + 'll' + else: + inner_init = str(self.default) + + if inner_init_only: + return inner_init + + outer_init = None + if self.allocation == 'STATIC': + if self.rules == 'REPEATED': + outer_init = '' + if not self.fixed_count: + outer_init += '0, ' + outer_init += '{' + outer_init += ', '.join([inner_init] * self.max_count) + outer_init += '}' + elif self.rules == 'OPTIONAL': + outer_init = 'false, ' + inner_init + else: + outer_init = inner_init + elif self.allocation == 'POINTER': + if self.rules == 'REPEATED': + outer_init = '0, NULL' + else: + outer_init = 'NULL' + elif self.allocation == 'CALLBACK': + if self.pbtype == 'EXTENSION': + outer_init = 'NULL' + else: + outer_init = '{{NULL}, NULL}' + + return outer_init + def default_decl(self, declaration_only = False): '''Return definition for this field's default value.''' if self.default is None: return None - ctype, default = self.ctype, self.default + ctype = self.ctype + default = self.get_initializer(False, True) array_decl = '' - + if self.pbtype == 'STRING': if self.allocation != 'STATIC': return None # Not implemented - array_decl = '[%d]' % self.max_size - default = str(self.default).encode('string_escape') - default = default.replace('"', '\\"') - default = '"' + default + '"' elif self.pbtype == 'BYTES': if self.allocation != 'STATIC': return None # Not implemented + elif self.pbtype == 'FIXED_LENGTH_BYTES': + if self.allocation != 'STATIC': + return None # Not implemented + array_decl = '[%d]' % self.max_size - data = self.default.decode('string_escape') - data = ['0x%02x' % ord(c) for c in data] - default = '{%d, {%s}}' % (len(data), ','.join(data)) - elif self.pbtype in ['FIXED32', 'UINT32']: - default += 'u' - elif self.pbtype in ['FIXED64', 'UINT64']: - default += 'ull' - elif self.pbtype in ['SFIXED64', 'INT64']: - default += 'll' - if declaration_only: return 'extern const %s %s_default%s;' % (ctype, self.struct_name + self.name, array_decl) else: return 'const %s %s_default%s = %s;' % (ctype, self.struct_name + self.name, array_decl, default) - + def tags(self): '''Return the #define for the tag number of this field.''' identifier = '%s_%s_tag' % (self.struct_name, self.name) return '#define %-40s %d\n' % (identifier, self.tag) - - def pb_field_t(self, prev_field_name): + + def pb_field_t(self, prev_field_name, union_index = None): '''Return the pb_field_t initializer to use in the constant array. - prev_field_name is the name of the previous field or None. + prev_field_name is the name of the previous field or None. For OneOf + unions, union_index is the index of this field inside the OneOf. ''' - result = ' PB_FIELD2(%3d, ' % self.tag + + if self.rules == 'ONEOF': + if self.anonymous: + result = ' PB_ANONYMOUS_ONEOF_FIELD(%s, ' % self.union_name + else: + result = ' PB_ONEOF_FIELD(%s, ' % self.union_name + elif self.fixed_count: + result = ' PB_REPEATED_FIXED_COUNT(' + else: + result = ' PB_FIELD(' + + result += '%3d, ' % self.tag result += '%-8s, ' % self.pbtype - result += '%s, ' % self.rules - result += '%-8s, ' % self.allocation - result += '%s, ' % ("FIRST" if not prev_field_name else "OTHER") + if not self.fixed_count: + result += '%s, ' % self.rules + result += '%-8s, ' % self.allocation + + if union_index is not None and union_index > 0: + result += 'UNION, ' + elif prev_field_name is None: + result += 'FIRST, ' + else: + result += 'OTHER, ' + result += '%s, ' % self.struct_name result += '%s, ' % self.name result += '%s, ' % (prev_field_name or self.name) - + if self.pbtype == 'MESSAGE': result += '&%s_fields)' % self.submsgname elif self.default is None: result += '0)' - elif self.pbtype in ['BYTES', 'STRING'] and self.allocation != 'STATIC': + elif self.pbtype in ['BYTES', 'STRING', 'FIXED_LENGTH_BYTES'] and self.allocation != 'STATIC': result += '0)' # Arbitrary size default values not implemented elif self.rules == 'OPTEXT': result += '0)' # Default value for extensions is not implemented else: result += '&%s_default)' % (self.struct_name + self.name) - + return result - + + def get_last_field_name(self): + return self.name + def largest_field_value(self): '''Determine if this field needs 16bit or 32bit pb_field_t structure to compile properly. Returns numeric value or a C-expression for assert.''' - if self.pbtype == 'MESSAGE': - if self.rules == 'REPEATED' and self.allocation == 'STATIC': - return 'pb_membersize(%s, %s[0])' % (self.struct_name, self.name) + check = [] + if self.pbtype == 'MESSAGE' and self.allocation == 'STATIC': + if self.rules == 'REPEATED': + check.append('pb_membersize(%s, %s[0])' % (self.struct_name, self.name)) + elif self.rules == 'ONEOF': + if self.anonymous: + check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name)) + else: + check.append('pb_membersize(%s, %s.%s)' % (self.struct_name, self.union_name, self.name)) else: - return 'pb_membersize(%s, %s)' % (self.struct_name, self.name) + check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name)) + elif self.pbtype == 'BYTES' and self.allocation == 'STATIC': + if self.max_size > 251: + check.append('pb_membersize(%s, %s)' % (self.struct_name, self.name)) - return max(self.tag, self.max_size, self.max_count) + return FieldMaxSize([self.tag, self.max_size, self.max_count], + check, + ('%s.%s' % (self.struct_name, self.name))) - def encoded_size(self, allmsgs): + def encoded_size(self, dependencies): '''Return the maximum size that this field can take when encoded, including the field tag. If the size cannot be determined, returns None.''' - + if self.allocation != 'STATIC': return None - + if self.pbtype == 'MESSAGE': - for msg in allmsgs: - if msg.name == self.submsgname: - encsize = msg.encoded_size(allmsgs) - if encsize is None: - return None # Submessage size is indeterminate - + encsize = None + if str(self.submsgname) in dependencies: + submsg = dependencies[str(self.submsgname)] + encsize = submsg.encoded_size(dependencies) + if encsize is not None: # Include submessage length prefix encsize += varint_max_size(encsize.upperlimit()) - break - else: - # Submessage cannot be found, this currently occurs when - # the submessage type is defined in a different file. + + if encsize is None: + # Submessage or its size cannot be found. + # This can occur if submessage is defined in different + # file, and it or its .options could not be found. # Instead of direct numeric value, reference the size that # has been #defined in the other file. encsize = EncodedSize(self.submsgname + 'size') @@ -398,20 +653,33 @@ class Field: # prefix size, though. encsize += 5 + elif self.pbtype in ['ENUM', 'UENUM']: + if str(self.ctype) in dependencies: + enumtype = dependencies[str(self.ctype)] + encsize = enumtype.encoded_size() + else: + # Conservative assumption + encsize = 10 + elif self.enc_size is None: raise RuntimeError("Could not determine encoded size for %s.%s" % (self.struct_name, self.name)) else: encsize = EncodedSize(self.enc_size) - + encsize += varint_max_size(self.tag << 3) # Tag + wire type if self.rules == 'REPEATED': # Decoders must be always able to handle unpacked arrays. # Therefore we have to reserve space for it, even though - # we emit packed arrays ourselves. + # we emit packed arrays ourselves. For length of 1, packed + # arrays are larger however so we need to add allowance + # for the length byte. encsize *= self.max_count - + + if self.max_count == 1: + encsize += 1 + return encsize @@ -433,17 +701,18 @@ class ExtensionRange(Field): self.default = None self.max_size = 0 self.max_count = 0 - + self.fixed_count = False + def __str__(self): return ' pb_extension_t *extensions;' - + def types(self): - return None - + return '' + def tags(self): return '' - - def encoded_size(self, allmsgs): + + def encoded_size(self, dependencies): # We exclude extensions from the count, because they cannot be known # until runtime. Other option would be to return None here, but this # way the value remains useful if extensions are not used. @@ -454,7 +723,7 @@ class ExtensionField(Field): self.fullname = struct_name + desc.name self.extendee_name = names_from_type_name(desc.extendee) Field.__init__(self, self.fullname + 'struct', desc, field_options) - + if self.rules != 'OPTIONAL': self.skip = True else: @@ -472,8 +741,9 @@ class ExtensionField(Field): msg = '/* Extension field %s was skipped because only "optional"\n' % self.fullname msg +=' type of extension fields is currently supported. */\n' return msg - - return 'extern const pb_extension_type_t %s;\n' % self.fullname + + return ('extern const pb_extension_type_t %s; /* field type: %s */\n' % + (self.fullname, str(self).strip())) def extension_def(self): '''Definition of the extension type in the .pb.c file''' @@ -495,6 +765,106 @@ class ExtensionField(Field): # --------------------------------------------------------------------------- +# Generation of oneofs (unions) +# --------------------------------------------------------------------------- + +class OneOf(Field): + def __init__(self, struct_name, oneof_desc): + self.struct_name = struct_name + self.name = oneof_desc.name + self.ctype = 'union' + self.pbtype = 'oneof' + self.fields = [] + self.allocation = 'ONEOF' + self.default = None + self.rules = 'ONEOF' + self.anonymous = False + + def add_field(self, field): + if field.allocation == 'CALLBACK': + raise Exception("Callback fields inside of oneof are not supported" + + " (field %s)" % field.name) + + field.union_name = self.name + field.rules = 'ONEOF' + field.anonymous = self.anonymous + self.fields.append(field) + self.fields.sort(key = lambda f: f.tag) + + # Sort by the lowest tag number inside union + self.tag = min([f.tag for f in self.fields]) + + def __str__(self): + result = '' + if self.fields: + result += ' pb_size_t which_' + self.name + ";\n" + result += ' union {\n' + for f in self.fields: + result += ' ' + str(f).replace('\n', '\n ') + '\n' + if self.anonymous: + result += ' };' + else: + result += ' } ' + self.name + ';' + return result + + def types(self): + return ''.join([f.types() for f in self.fields]) + + def get_dependencies(self): + deps = [] + for f in self.fields: + deps += f.get_dependencies() + return deps + + def get_initializer(self, null_init): + return '0, {' + self.fields[0].get_initializer(null_init) + '}' + + def default_decl(self, declaration_only = False): + return None + + def tags(self): + return ''.join([f.tags() for f in self.fields]) + + def pb_field_t(self, prev_field_name): + parts = [] + for union_index, field in enumerate(self.fields): + parts.append(field.pb_field_t(prev_field_name, union_index)) + return ',\n'.join(parts) + + def get_last_field_name(self): + if self.anonymous: + return self.fields[-1].name + else: + return self.name + '.' + self.fields[-1].name + + def largest_field_value(self): + largest = FieldMaxSize() + for f in self.fields: + largest.extend(f.largest_field_value()) + return largest + + def encoded_size(self, dependencies): + '''Returns the size of the largest oneof field.''' + largest = EncodedSize(0) + symbols = set() + for f in self.fields: + size = EncodedSize(f.encoded_size(dependencies)) + if size.value is None: + return None + elif size.symbols: + symbols.add(EncodedSize(f.submsgname + 'size').symbols[0]) + elif size.value > largest.value: + largest = size + + if not symbols: + return largest + + symbols = list(symbols) + symbols.append(str(largest)) + max_size = lambda x, y: '({0} > {1} ? {0} : {1})'.format(x, y) + return reduce(max_size, symbols) + +# --------------------------------------------------------------------------- # Generation of messages (structures) # --------------------------------------------------------------------------- @@ -503,56 +873,94 @@ class Message: def __init__(self, names, desc, message_options): self.name = names self.fields = [] - + self.oneofs = {} + no_unions = [] + + if message_options.msgid: + self.msgid = message_options.msgid + + if hasattr(desc, 'oneof_decl'): + for i, f in enumerate(desc.oneof_decl): + oneof_options = get_nanopb_suboptions(desc, message_options, self.name + f.name) + if oneof_options.no_unions: + no_unions.append(i) # No union, but add fields normally + elif oneof_options.type == nanopb_pb2.FT_IGNORE: + pass # No union and skip fields also + else: + oneof = OneOf(self.name, f) + if oneof_options.anonymous_oneof: + oneof.anonymous = True + self.oneofs[i] = oneof + self.fields.append(oneof) + else: + sys.stderr.write('Note: This Python protobuf library has no OneOf support\n') + for f in desc.field: field_options = get_nanopb_suboptions(f, message_options, self.name + f.name) - if field_options.type != nanopb_pb2.FT_IGNORE: - self.fields.append(Field(self.name, f, field_options)) - + if field_options.type == nanopb_pb2.FT_IGNORE: + continue + + field = Field(self.name, f, field_options) + if (hasattr(f, 'oneof_index') and + f.HasField('oneof_index') and + f.oneof_index not in no_unions): + if f.oneof_index in self.oneofs: + self.oneofs[f.oneof_index].add_field(field) + else: + self.fields.append(field) + if len(desc.extension_range) > 0: field_options = get_nanopb_suboptions(desc, message_options, self.name + 'extensions') range_start = min([r.start for r in desc.extension_range]) if field_options.type != nanopb_pb2.FT_IGNORE: self.fields.append(ExtensionRange(self.name, range_start, field_options)) - + self.packed = message_options.packed_struct self.ordered_fields = self.fields[:] self.ordered_fields.sort() def get_dependencies(self): '''Get list of type names that this structure refers to.''' - return [str(field.ctype) for field in self.fields] - + deps = [] + for f in self.fields: + deps += f.get_dependencies() + return deps + def __str__(self): result = 'typedef struct _%s {\n' % self.name if not self.ordered_fields: # Empty structs are not allowed in C standard. # Therefore add a dummy field if an empty message occurs. - result += ' uint8_t dummy_field;' + result += ' char dummy_field;' result += '\n'.join([str(f) for f in self.ordered_fields]) + result += '\n/* @@protoc_insertion_point(struct:%s) */' % self.name result += '\n}' - + if self.packed: result += ' pb_packed' - + result += ' %s;' % self.name - + if self.packed: result = 'PB_PACKED_STRUCT_START\n' + result result += '\nPB_PACKED_STRUCT_END' - + return result - + def types(self): - result = "" - for field in self.fields: - types = field.types() - if types is not None: - result += types + '\n' - return result - + return ''.join([f.types() for f in self.fields]) + + def get_initializer(self, null_init): + if not self.ordered_fields: + return '{0}' + + parts = [] + for field in self.ordered_fields: + parts.append(field.get_initializer(null_init)) + return '{' + ', '.join(parts) + '}' + def default_decl(self, declaration_only = False): result = "" for field in self.fields: @@ -561,33 +969,51 @@ class Message: result += default + '\n' return result + def count_required_fields(self): + '''Returns number of required fields inside this message''' + count = 0 + for f in self.fields: + if not isinstance(f, OneOf): + if f.rules == 'REQUIRED': + count += 1 + return count + + def count_all_fields(self): + count = 0 + for f in self.fields: + if isinstance(f, OneOf): + count += len(f.fields) + else: + count += 1 + return count + def fields_declaration(self): - result = 'extern const pb_field_t %s_fields[%d];' % (self.name, len(self.fields) + 1) + result = 'extern const pb_field_t %s_fields[%d];' % (self.name, self.count_all_fields() + 1) return result def fields_definition(self): - result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, len(self.fields) + 1) - + result = 'const pb_field_t %s_fields[%d] = {\n' % (self.name, self.count_all_fields() + 1) + prev = None for field in self.ordered_fields: result += field.pb_field_t(prev) result += ',\n' - prev = field.name - + prev = field.get_last_field_name() + result += ' PB_LAST_FIELD\n};' return result - def encoded_size(self, allmsgs): + def encoded_size(self, dependencies): '''Return the maximum size that this message can take when encoded. If the size cannot be determined, returns None. ''' size = EncodedSize(0) for field in self.fields: - fsize = field.encoded_size(allmsgs) + fsize = field.encoded_size(dependencies) if fsize is None: return None size += fsize - + return size @@ -595,18 +1021,17 @@ class Message: # Processing of entire .proto files # --------------------------------------------------------------------------- - def iterate_messages(desc, names = Names()): '''Recursively find all messages. For each, yield name, DescriptorProto.''' if hasattr(desc, 'message_type'): submsgs = desc.message_type else: submsgs = desc.nested_type - + for submsg in submsgs: sub_names = names + submsg.name yield sub_names, submsg - + for x in iterate_messages(submsg, sub_names): yield x @@ -621,61 +1046,22 @@ def iterate_extensions(desc, names = Names()): for extension in subdesc.extension: yield subname, extension -def parse_file(fdesc, file_options): - '''Takes a FileDescriptorProto and returns tuple (enums, messages, extensions).''' - - enums = [] - messages = [] - extensions = [] - - if fdesc.package: - base_name = Names(fdesc.package.split('.')) - else: - base_name = Names() - - for enum in fdesc.enum_type: - enum_options = get_nanopb_suboptions(enum, file_options, base_name + enum.name) - enums.append(Enum(base_name, enum, enum_options)) - - for names, message in iterate_messages(fdesc, base_name): - message_options = get_nanopb_suboptions(message, file_options, names) - messages.append(Message(names, message, message_options)) - for enum in message.enum_type: - enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name) - enums.append(Enum(names, enum, enum_options)) - - for names, extension in iterate_extensions(fdesc, base_name): - field_options = get_nanopb_suboptions(extension, file_options, names) - if field_options.type != nanopb_pb2.FT_IGNORE: - extensions.append(ExtensionField(names, extension, field_options)) - - # Fix field default values where enum short names are used. - for enum in enums: - if not enum.options.long_names: - for message in messages: - for field in message.fields: - if field.default in enum.value_longnames: - idx = enum.value_longnames.index(field.default) - field.default = enum.values[idx][0] - - return enums, messages, extensions - def toposort2(data): '''Topological sort. From http://code.activestate.com/recipes/577413-topological-sort/ This function is under the MIT license. ''' - for k, v in data.items(): + for k, v in list(data.items()): v.discard(k) # Ignore self dependencies - extra_items_in_deps = reduce(set.union, data.values(), set()) - set(data.keys()) + extra_items_in_deps = reduce(set.union, list(data.values()), set()) - set(data.keys()) data.update(dict([(item, set()) for item in extra_items_in_deps])) while True: - ordered = set(item for item,dep in data.items() if not dep) + ordered = set(item for item,dep in list(data.items()) if not dep) if not ordered: break for item in sorted(ordered): yield item - data = dict([(item, (dep - ordered)) for item,dep in data.items() + data = dict([(item, (dep - ordered)) for item,dep in list(data.items()) if item not in ordered]) assert not data, "A cyclic dependency exists amongst %r" % data @@ -686,7 +1072,7 @@ def sort_dependencies(messages): for message in messages: dependencies[str(message.name)] = set(message.get_dependencies()) message_by_name[str(message.name)] = message - + for msgname in toposort2(dependencies): if msgname in message_by_name: yield message_by_name[msgname] @@ -701,185 +1087,311 @@ def make_identifier(headername): result += '_' return result -def generate_header(dependencies, headername, enums, messages, extensions, options): - '''Generate content for a header file. - Generates strings, which should be concatenated and stored to file. - ''' - - yield '/* Automatically generated nanopb header */\n' - if options.notimestamp: - yield '/* Generated by %s */\n\n' % (nanopb_version) - else: - yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) - - symbol = make_identifier(headername) - yield '#ifndef _PB_%s_\n' % symbol - yield '#define _PB_%s_\n' % symbol - try: - yield options.libformat % ('pb.h') - except TypeError: - # no %s specified - use whatever was passed in as options.libformat - yield options.libformat - yield '\n' - - for dependency in dependencies: - noext = os.path.splitext(dependency)[0] - yield options.genformat % (noext + '.' + options.extension + '.h') +class ProtoFile: + def __init__(self, fdesc, file_options): + '''Takes a FileDescriptorProto and parses it.''' + self.fdesc = fdesc + self.file_options = file_options + self.dependencies = {} + self.parse() + + # Some of types used in this file probably come from the file itself. + # Thus it has implicit dependency on itself. + self.add_dependency(self) + + def parse(self): + self.enums = [] + self.messages = [] + self.extensions = [] + + if self.fdesc.package: + base_name = Names(self.fdesc.package.split('.')) + else: + base_name = Names() + + for enum in self.fdesc.enum_type: + enum_options = get_nanopb_suboptions(enum, self.file_options, base_name + enum.name) + self.enums.append(Enum(base_name, enum, enum_options)) + + for names, message in iterate_messages(self.fdesc, base_name): + message_options = get_nanopb_suboptions(message, self.file_options, names) + + if message_options.skip_message: + continue + + self.messages.append(Message(names, message, message_options)) + for enum in message.enum_type: + enum_options = get_nanopb_suboptions(enum, message_options, names + enum.name) + self.enums.append(Enum(names, enum, enum_options)) + + for names, extension in iterate_extensions(self.fdesc, base_name): + field_options = get_nanopb_suboptions(extension, self.file_options, names + extension.name) + if field_options.type != nanopb_pb2.FT_IGNORE: + self.extensions.append(ExtensionField(names, extension, field_options)) + + def add_dependency(self, other): + for enum in other.enums: + self.dependencies[str(enum.names)] = enum + + for msg in other.messages: + self.dependencies[str(msg.name)] = msg + + # Fix field default values where enum short names are used. + for enum in other.enums: + if not enum.options.long_names: + for message in self.messages: + for field in message.fields: + if field.default in enum.value_longnames: + idx = enum.value_longnames.index(field.default) + field.default = enum.values[idx][0] + + # Fix field data types where enums have negative values. + for enum in other.enums: + if not enum.has_negative(): + for message in self.messages: + for field in message.fields: + if field.pbtype == 'ENUM' and field.ctype == enum.names: + field.pbtype = 'UENUM' + + def generate_header(self, includes, headername, options): + '''Generate content for a header file. + Generates strings, which should be concatenated and stored to file. + ''' + + yield '/* Automatically generated nanopb header */\n' + if options.notimestamp: + yield '/* Generated by %s */\n\n' % (nanopb_version) + else: + yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) + + if self.fdesc.package: + symbol = make_identifier(self.fdesc.package + '_' + headername) + else: + symbol = make_identifier(headername) + yield '#ifndef PB_%s_INCLUDED\n' % symbol + yield '#define PB_%s_INCLUDED\n' % symbol + try: + yield options.libformat % ('pb.h') + except TypeError: + # no %s specified - use whatever was passed in as options.libformat + yield options.libformat yield '\n' - yield '#ifdef __cplusplus\n' - yield 'extern "C" {\n' - yield '#endif\n\n' - - yield '/* Enum definitions */\n' - for enum in enums: - yield str(enum) + '\n\n' - - yield '/* Struct definitions */\n' - for msg in sort_dependencies(messages): - yield msg.types() - yield str(msg) + '\n\n' - - if extensions: - yield '/* Extensions */\n' - for extension in extensions: - yield extension.extension_decl() + for incfile in includes: + noext = os.path.splitext(incfile)[0] + yield options.genformat % (noext + options.extension + options.header_extension) + yield '\n' + + yield '/* @@protoc_insertion_point(includes) */\n' + + yield '#if PB_PROTO_HEADER_VERSION != 30\n' + yield '#error Regenerate this file with the current version of nanopb generator.\n' + yield '#endif\n' yield '\n' - - yield '/* Default values for struct fields */\n' - for msg in messages: - yield msg.default_decl(True) - yield '\n' - - yield '/* Field tags (for use in manual encoding/decoding) */\n' - for msg in sort_dependencies(messages): - for field in msg.fields: - yield field.tags() - for extension in extensions: - yield extension.tags() - yield '\n' - - yield '/* Struct field encoding specification for nanopb */\n' - for msg in messages: - yield msg.fields_declaration() + '\n' - yield '\n' - - yield '/* Maximum encoded size of messages (where known) */\n' - for msg in messages: - msize = msg.encoded_size(messages) - if msize is not None: - identifier = '%s_size' % msg.name - yield '#define %-40s %s\n' % (identifier, msize) - yield '\n' - - yield '#ifdef __cplusplus\n' - yield '} /* extern "C" */\n' - yield '#endif\n' - - # End of header - yield '\n#endif\n' - -def generate_source(headername, enums, messages, extensions, options): - '''Generate content for a source file.''' - - yield '/* Automatically generated nanopb constant definitions */\n' - if options.notimestamp: - yield '/* Generated by %s */\n\n' % (nanopb_version) - else: - yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) - yield options.genformat % (headername) - yield '\n' - - for msg in messages: - yield msg.default_decl(False) - - yield '\n\n' - - for msg in messages: - yield msg.fields_definition() + '\n\n' - - for ext in extensions: - yield ext.extension_def() + '\n' - - # Add checks for numeric limits - if messages: - count_required_fields = lambda m: len([f for f in msg.fields if f.rules == 'REQUIRED']) - largest_msg = max(messages, key = count_required_fields) - largest_count = count_required_fields(largest_msg) - if largest_count > 64: - yield '\n/* Check that missing required fields will be properly detected */\n' - yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count - yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name - yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count - yield '#endif\n' - - worst = 0 - worst_field = '' - checks = [] - checks_msgnames = [] - for msg in messages: - checks_msgnames.append(msg.name) - for field in msg.fields: - status = field.largest_field_value() - if isinstance(status, (str, unicode)): - checks.append(status) - elif status > worst: - worst = status - worst_field = str(field.struct_name) + '.' + str(field.name) - - if worst > 255 or checks: - yield '\n/* Check that field information fits in pb_field_t */\n' - - if worst > 65535 or checks: - yield '#if !defined(PB_FIELD_32BIT)\n' - if worst > 65535: - yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field - else: - assertion = ' && '.join(str(c) + ' < 65536' for c in checks) - msgs = '_'.join(str(n) for n in checks_msgnames) - yield '/* If you get an error here, it means that you need to define PB_FIELD_32BIT\n' - yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n' - yield ' * \n' - yield ' * The reason you need to do this is that some of your messages contain tag\n' - yield ' * numbers or field sizes that are larger than what can fit in 8 or 16 bit\n' - yield ' * field descriptors.\n' - yield ' */\n' - yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) - yield '#endif\n\n' - - if worst < 65536: - yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n' - if worst > 255: - yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field - else: - assertion = ' && '.join(str(c) + ' < 256' for c in checks) - msgs = '_'.join(str(n) for n in checks_msgnames) - yield '/* If you get an error here, it means that you need to define PB_FIELD_16BIT\n' - yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n' - yield ' * \n' - yield ' * The reason you need to do this is that some of your messages contain tag\n' - yield ' * numbers or field sizes that are larger than what can fit in the default\n' - yield ' * 8 bit descriptors.\n' - yield ' */\n' - yield 'STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) + + yield '#ifdef __cplusplus\n' + yield 'extern "C" {\n' + yield '#endif\n\n' + + if self.enums: + yield '/* Enum definitions */\n' + for enum in self.enums: + yield str(enum) + '\n\n' + + if self.messages: + yield '/* Struct definitions */\n' + for msg in sort_dependencies(self.messages): + yield msg.types() + yield str(msg) + '\n\n' + + if self.extensions: + yield '/* Extensions */\n' + for extension in self.extensions: + yield extension.extension_decl() + yield '\n' + + if self.messages: + yield '/* Default values for struct fields */\n' + for msg in self.messages: + yield msg.default_decl(True) + yield '\n' + + yield '/* Initializer values for message structs */\n' + for msg in self.messages: + identifier = '%s_init_default' % msg.name + yield '#define %-40s %s\n' % (identifier, msg.get_initializer(False)) + for msg in self.messages: + identifier = '%s_init_zero' % msg.name + yield '#define %-40s %s\n' % (identifier, msg.get_initializer(True)) + yield '\n' + + yield '/* Field tags (for use in manual encoding/decoding) */\n' + for msg in sort_dependencies(self.messages): + for field in msg.fields: + yield field.tags() + for extension in self.extensions: + yield extension.tags() + yield '\n' + + yield '/* Struct field encoding specification for nanopb */\n' + for msg in self.messages: + yield msg.fields_declaration() + '\n' + yield '\n' + + yield '/* Maximum encoded size of messages (where known) */\n' + for msg in self.messages: + msize = msg.encoded_size(self.dependencies) + identifier = '%s_size' % msg.name + if msize is not None: + yield '#define %-40s %s\n' % (identifier, msize) + else: + yield '/* %s depends on runtime parameters */\n' % identifier + yield '\n' + + yield '/* Message IDs (where set with "msgid" option) */\n' + + yield '#ifdef PB_MSGID\n' + for msg in self.messages: + if hasattr(msg,'msgid'): + yield '#define PB_MSG_%d %s\n' % (msg.msgid, msg.name) + yield '\n' + + symbol = make_identifier(headername.split('.')[0]) + yield '#define %s_MESSAGES \\\n' % symbol + + for msg in self.messages: + m = "-1" + msize = msg.encoded_size(self.dependencies) + if msize is not None: + m = msize + if hasattr(msg,'msgid'): + yield '\tPB_MSG(%d,%s,%s) \\\n' % (msg.msgid, m, msg.name) + yield '\n' + + for msg in self.messages: + if hasattr(msg,'msgid'): + yield '#define %s_msgid %d\n' % (msg.name, msg.msgid) + yield '\n' + yield '#endif\n\n' - - # Add check for sizeof(double) - has_double = False - for msg in messages: - for field in msg.fields: - if field.ctype == 'double': - has_double = True - - if has_double: + + yield '#ifdef __cplusplus\n' + yield '} /* extern "C" */\n' + yield '#endif\n' + + # End of header + yield '/* @@protoc_insertion_point(eof) */\n' + yield '\n#endif\n' + + def generate_source(self, headername, options): + '''Generate content for a source file.''' + + yield '/* Automatically generated nanopb constant definitions */\n' + if options.notimestamp: + yield '/* Generated by %s */\n\n' % (nanopb_version) + else: + yield '/* Generated by %s at %s. */\n\n' % (nanopb_version, time.asctime()) + yield options.genformat % (headername) yield '\n' - yield '/* On some platforms (such as AVR), double is really float.\n' - yield ' * These are not directly supported by nanopb, but see example_avr_double.\n' - yield ' * To get rid of this error, remove any double fields from your .proto.\n' - yield ' */\n' - yield 'STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n' - - yield '\n' + yield '/* @@protoc_insertion_point(includes) */\n' + + yield '#if PB_PROTO_HEADER_VERSION != 30\n' + yield '#error Regenerate this file with the current version of nanopb generator.\n' + yield '#endif\n' + yield '\n' + + for msg in self.messages: + yield msg.default_decl(False) + + yield '\n\n' + + for msg in self.messages: + yield msg.fields_definition() + '\n\n' + + for ext in self.extensions: + yield ext.extension_def() + '\n' + + for enum in self.enums: + yield enum.enum_to_string_definition() + '\n' + + # Add checks for numeric limits + if self.messages: + largest_msg = max(self.messages, key = lambda m: m.count_required_fields()) + largest_count = largest_msg.count_required_fields() + if largest_count > 64: + yield '\n/* Check that missing required fields will be properly detected */\n' + yield '#if PB_MAX_REQUIRED_FIELDS < %d\n' % largest_count + yield '#error Properly detecting missing required fields in %s requires \\\n' % largest_msg.name + yield ' setting PB_MAX_REQUIRED_FIELDS to %d or more.\n' % largest_count + yield '#endif\n' + + max_field = FieldMaxSize() + checks_msgnames = [] + for msg in self.messages: + checks_msgnames.append(msg.name) + for field in msg.fields: + max_field.extend(field.largest_field_value()) + for field in self.extensions: + max_field.extend(field.largest_field_value()) + + worst = max_field.worst + worst_field = max_field.worst_field + checks = max_field.checks + + if worst > 255 or checks: + yield '\n/* Check that field information fits in pb_field_t */\n' + + if worst > 65535 or checks: + yield '#if !defined(PB_FIELD_32BIT)\n' + if worst > 65535: + yield '#error Field descriptor for %s is too large. Define PB_FIELD_32BIT to fix this.\n' % worst_field + else: + assertion = ' && '.join(str(c) + ' < 65536' for c in checks) + msgs = '_'.join(str(n) for n in checks_msgnames) + yield '/* If you get an error here, it means that you need to define PB_FIELD_32BIT\n' + yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n' + yield ' * \n' + yield ' * The reason you need to do this is that some of your messages contain tag\n' + yield ' * numbers or field sizes that are larger than what can fit in 8 or 16 bit\n' + yield ' * field descriptors.\n' + yield ' */\n' + yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) + yield '#endif\n\n' + + if worst < 65536: + yield '#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT)\n' + if worst > 255: + yield '#error Field descriptor for %s is too large. Define PB_FIELD_16BIT to fix this.\n' % worst_field + else: + assertion = ' && '.join(str(c) + ' < 256' for c in checks) + msgs = '_'.join(str(n) for n in checks_msgnames) + yield '/* If you get an error here, it means that you need to define PB_FIELD_16BIT\n' + yield ' * compile-time option. You can do that in pb.h or on compiler command line.\n' + yield ' * \n' + yield ' * The reason you need to do this is that some of your messages contain tag\n' + yield ' * numbers or field sizes that are larger than what can fit in the default\n' + yield ' * 8 bit descriptors.\n' + yield ' */\n' + yield 'PB_STATIC_ASSERT((%s), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_%s)\n'%(assertion,msgs) + yield '#endif\n\n' + + # Add check for sizeof(double) + has_double = False + for msg in self.messages: + for field in msg.fields: + if field.ctype == 'double': + has_double = True + + if has_double: + yield '\n' + yield '/* On some platforms (such as AVR), double is really float.\n' + yield ' * These are not directly supported by nanopb, but see example_avr_double.\n' + yield ' * To get rid of this error, remove any double fields from your .proto.\n' + yield ' */\n' + yield 'PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES)\n' + + yield '\n' + yield '/* @@protoc_insertion_point(eof) */\n' # --------------------------------------------------------------------------- # Options parsing for the .proto files @@ -892,14 +1404,32 @@ def read_options_file(infile): [(namemask, options), ...] ''' results = [] - for line in infile: + data = infile.read() + data = re.sub('/\*.*?\*/', '', data, flags = re.MULTILINE) + data = re.sub('//.*?$', '', data, flags = re.MULTILINE) + data = re.sub('#.*?$', '', data, flags = re.MULTILINE) + for i, line in enumerate(data.split('\n')): line = line.strip() - if not line or line.startswith('//') or line.startswith('#'): + if not line: continue - + parts = line.split(None, 1) + + if len(parts) < 2: + sys.stderr.write("%s:%d: " % (infile.name, i + 1) + + "Option lines should have space between field name and options. " + + "Skipping line: '%s'\n" % line) + continue + opts = nanopb_pb2.NanoPBOptions() - text_format.Merge(parts[1], opts) + + try: + text_format.Merge(parts[1], opts) + except Exception as e: + sys.stderr.write("%s:%d: " % (infile.name, i + 1) + + "Unparseable option line: '%s'. " % line + + "Error: %s\n" % str(e)) + continue results.append((parts[0], opts)) return results @@ -914,14 +1444,17 @@ def get_nanopb_suboptions(subdesc, options, name): '''Get copy of options, and merge information from subdesc.''' new_options = nanopb_pb2.NanoPBOptions() new_options.CopyFrom(options) - + + if hasattr(subdesc, 'syntax') and subdesc.syntax == "proto3": + new_options.proto3 = True + # Handle options defined in a separate file dotname = '.'.join(name.parts) for namemask, options in Globals.separate_options: if fnmatch(dotname, namemask): Globals.matched_namemasks.add(namemask) new_options.MergeFrom(options) - + # Handle options defined in .proto if isinstance(subdesc.options, descriptor.FieldOptions): ext_type = nanopb_pb2.nanopb @@ -933,15 +1466,15 @@ def get_nanopb_suboptions(subdesc, options, name): ext_type = nanopb_pb2.nanopb_enumopt else: raise Exception("Unknown options type") - + if subdesc.options.HasExtension(ext_type): ext = subdesc.options.Extensions[ext_type] new_options.MergeFrom(ext) - + if Globals.verbose_options: sys.stderr.write("Options for " + dotname + ": ") sys.stderr.write(text_format.MessageToString(new_options) + "\n") - + return new_options @@ -950,7 +1483,7 @@ def get_nanopb_suboptions(subdesc, options, name): # --------------------------------------------------------------------------- import sys -import os.path +import os.path from optparse import OptionParser optparser = OptionParser( @@ -959,10 +1492,20 @@ optparser = OptionParser( "Output will be written to file.pb.h and file.pb.c.") optparser.add_option("-x", dest="exclude", metavar="FILE", action="append", default=[], help="Exclude file from generated #include list.") -optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default="pb", - help="Set extension to use instead of 'pb' for generated files. [default: %default]") +optparser.add_option("-e", "--extension", dest="extension", metavar="EXTENSION", default=".pb", + help="Set extension to use instead of '.pb' for generated files. [default: %default]") +optparser.add_option("-H", "--header-extension", dest="header_extension", metavar="EXTENSION", default=".h", + help="Set extension to use for generated header files. [default: %default]") +optparser.add_option("-S", "--source-extension", dest="source_extension", metavar="EXTENSION", default=".c", + help="Set extension to use for generated source files. [default: %default]") optparser.add_option("-f", "--options-file", dest="options_file", metavar="FILE", default="%s.options", help="Set name of a separate generator options file.") +optparser.add_option("-I", "--options-path", dest="options_path", metavar="DIR", + action="append", default = [], + help="Search for .options files additionally in this path") +optparser.add_option("-D", "--output-dir", dest="output_dir", + metavar="OUTPUTDIR", default=None, + help="Output directory of .pb.h and .pb.c files") optparser.add_option("-Q", "--generated-include-format", dest="genformat", metavar="FORMAT", default='#include "%s"\n', help="Set format string to use for including other .pb.h files. [default: %default]") @@ -978,134 +1521,193 @@ optparser.add_option("-v", "--verbose", dest="verbose", action="store_true", def optparser.add_option("-s", dest="settings", metavar="OPTION:VALUE", action="append", default=[], help="Set generator option (max_size, max_count etc.).") -def process_file(filename, fdesc, options): - '''Process a single file. - filename: The full path to the .proto or .pb source file, as string. - fdesc: The loaded FileDescriptorSet, or None to read from the input file. - options: Command line options as they come from OptionsParser. - - Returns a dict: - {'headername': Name of header file, - 'headerdata': Data for the .h header file, - 'sourcename': Name of the source code file, - 'sourcedata': Data for the .c source code file - } - ''' +def parse_file(filename, fdesc, options): + '''Parse a single file. Returns a ProtoFile instance.''' toplevel_options = nanopb_pb2.NanoPBOptions() for s in options.settings: text_format.Merge(s, toplevel_options) - + if not fdesc: data = open(filename, 'rb').read() fdesc = descriptor.FileDescriptorSet.FromString(data).file[0] - + # Check if there is a separate .options file + had_abspath = False try: optfilename = options.options_file % os.path.splitext(filename)[0] except TypeError: # No %s specified, use the filename as-is optfilename = options.options_file - - if os.path.isfile(optfilename): - if options.verbose: - sys.stderr.write('Reading options from ' + optfilename + '\n') - - Globals.separate_options = read_options_file(open(optfilename, "rU")) + had_abspath = True + + paths = ['.'] + options.options_path + for p in paths: + if os.path.isfile(os.path.join(p, optfilename)): + optfilename = os.path.join(p, optfilename) + if options.verbose: + sys.stderr.write('Reading options from ' + optfilename + '\n') + Globals.separate_options = read_options_file(open(optfilename, "rU")) + break else: + # If we are given a full filename and it does not exist, give an error. + # However, don't give error when we automatically look for .options file + # with the same name as .proto. + if options.verbose or had_abspath: + sys.stderr.write('Options file not found: ' + optfilename + '\n') Globals.separate_options = [] + Globals.matched_namemasks = set() - + # Parse the file file_options = get_nanopb_suboptions(fdesc, toplevel_options, Names([filename])) - enums, messages, extensions = parse_file(fdesc, file_options) - + f = ProtoFile(fdesc, file_options) + f.optfilename = optfilename + + return f + +def process_file(filename, fdesc, options, other_files = {}): + '''Process a single file. + filename: The full path to the .proto or .pb source file, as string. + fdesc: The loaded FileDescriptorSet, or None to read from the input file. + options: Command line options as they come from OptionsParser. + + Returns a dict: + {'headername': Name of header file, + 'headerdata': Data for the .h header file, + 'sourcename': Name of the source code file, + 'sourcedata': Data for the .c source code file + } + ''' + f = parse_file(filename, fdesc, options) + + # Provide dependencies if available + for dep in f.fdesc.dependency: + if dep in other_files: + f.add_dependency(other_files[dep]) + # Decide the file names noext = os.path.splitext(filename)[0] - headername = noext + '.' + options.extension + '.h' - sourcename = noext + '.' + options.extension + '.c' + headername = noext + options.extension + options.header_extension + sourcename = noext + options.extension + options.source_extension headerbasename = os.path.basename(headername) - + # List of .proto files that should not be included in the C header file # even if they are mentioned in the source .proto. excludes = ['nanopb.proto', 'google/protobuf/descriptor.proto'] + options.exclude - dependencies = [d for d in fdesc.dependency if d not in excludes] - - headerdata = ''.join(generate_header(dependencies, headerbasename, enums, - messages, extensions, options)) + includes = [d for d in f.fdesc.dependency if d not in excludes] - sourcedata = ''.join(generate_source(headerbasename, enums, - messages, extensions, options)) + headerdata = ''.join(f.generate_header(includes, headerbasename, options)) + sourcedata = ''.join(f.generate_source(headerbasename, options)) # Check if there were any lines in .options that did not match a member unmatched = [n for n,o in Globals.separate_options if n not in Globals.matched_namemasks] if unmatched and not options.quiet: - sys.stderr.write("Following patterns in " + optfilename + " did not match any fields: " + sys.stderr.write("Following patterns in " + f.optfilename + " did not match any fields: " + ', '.join(unmatched) + "\n") if not Globals.verbose_options: sys.stderr.write("Use protoc --nanopb-out=-v:. to see a list of the field names.\n") return {'headername': headername, 'headerdata': headerdata, 'sourcename': sourcename, 'sourcedata': sourcedata} - + def main_cli(): '''Main function when invoked directly from the command line.''' - + options, filenames = optparser.parse_args() - + if not filenames: optparser.print_help() sys.exit(1) - + if options.quiet: options.verbose = False + if options.output_dir and not os.path.exists(options.output_dir): + optparser.print_help() + sys.stderr.write("\noutput_dir does not exist: %s\n" % options.output_dir) + sys.exit(1) + + if options.verbose: + sys.stderr.write('Google Python protobuf library imported from %s, version %s\n' + % (google.protobuf.__file__, google.protobuf.__version__)) + Globals.verbose_options = options.verbose - for filename in filenames: results = process_file(filename, None, options) - + + base_dir = options.output_dir or '' + to_write = [ + (os.path.join(base_dir, results['headername']), results['headerdata']), + (os.path.join(base_dir, results['sourcename']), results['sourcedata']), + ] + if not options.quiet: - sys.stderr.write("Writing to " + results['headername'] + " and " - + results['sourcename'] + "\n") - - open(results['headername'], 'w').write(results['headerdata']) - open(results['sourcename'], 'w').write(results['sourcedata']) + paths = " and ".join([x[0] for x in to_write]) + sys.stderr.write("Writing to %s\n" % paths) + + for path, data in to_write: + with open(path, 'w') as f: + f.write(data) def main_plugin(): '''Main function when invoked as a protoc plugin.''' - import sys + import io, sys if sys.platform == "win32": import os, msvcrt # Set stdin and stdout to binary mode msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) msvcrt.setmode(sys.stdout.fileno(), os.O_BINARY) - - data = sys.stdin.read() + + data = io.open(sys.stdin.fileno(), "rb").read() + request = plugin_pb2.CodeGeneratorRequest.FromString(data) - + + try: + # Versions of Python prior to 2.7.3 do not support unicode + # input to shlex.split(). Try to convert to str if possible. + params = str(request.parameter) + except UnicodeEncodeError: + params = request.parameter + import shlex - args = shlex.split(request.parameter) + args = shlex.split(params) options, dummy = optparser.parse_args(args) - + Globals.verbose_options = options.verbose - + + if options.verbose: + sys.stderr.write('Google Python protobuf library imported from %s, version %s\n' + % (google.protobuf.__file__, google.protobuf.__version__)) + response = plugin_pb2.CodeGeneratorResponse() - + + # Google's protoc does not currently indicate the full path of proto files. + # Instead always add the main file path to the search dirs, that works for + # the common case. + import os.path + options.options_path.append(os.path.dirname(request.file_to_generate[0])) + + # Process any include files first, in order to have them + # available as dependencies + other_files = {} + for fdesc in request.proto_file: + other_files[fdesc.name] = parse_file(fdesc.name, fdesc, options) + for filename in request.file_to_generate: for fdesc in request.proto_file: if fdesc.name == filename: - results = process_file(filename, fdesc, options) - + results = process_file(filename, fdesc, options, other_files) + f = response.file.add() f.name = results['headername'] f.content = results['headerdata'] f = response.file.add() f.name = results['sourcename'] - f.content = results['sourcedata'] - - sys.stdout.write(response.SerializeToString()) + f.content = results['sourcedata'] + + io.open(sys.stdout.fileno(), "wb").write(response.SerializeToString()) if __name__ == '__main__': # Check if we are running as a plugin under protoc @@ -1113,4 +1715,3 @@ if __name__ == '__main__': main_plugin() else: main_cli() - diff --git a/generator/proto/Makefile b/generator/proto/Makefile index 032392c..89bfe52 100644 --- a/generator/proto/Makefile +++ b/generator/proto/Makefile @@ -1,4 +1,4 @@ all: nanopb_pb2.py plugin_pb2.py %_pb2.py: %.proto - aprotoc --python_out=. $< + protoc --python_out=. $< diff --git a/generator/proto/google/protobuf/descriptor.proto b/generator/proto/google/protobuf/descriptor.proto index a785f79..8697a50 100644 --- a/generator/proto/google/protobuf/descriptor.proto +++ b/generator/proto/google/protobuf/descriptor.proto @@ -1,6 +1,6 @@ // Protocol Buffers - Google's data interchange format // Copyright 2008 Google Inc. All rights reserved. -// http://code.google.com/p/protobuf/ +// https://developers.google.com/protocol-buffers/ // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are @@ -37,10 +37,15 @@ // without any other information (e.g. without reading its imports). +syntax = "proto2"; package google.protobuf; +option go_package = "github.com/golang/protobuf/protoc-gen-go/descriptor;descriptor"; option java_package = "com.google.protobuf"; option java_outer_classname = "DescriptorProtos"; +option csharp_namespace = "Google.Protobuf.Reflection"; +option objc_class_prefix = "GPB"; +option cc_enable_arenas = true; // descriptor.proto must be optimized for speed because reflection-based // algorithms don't work during bootstrapping. @@ -74,10 +79,14 @@ message FileDescriptorProto { optional FileOptions options = 8; // This field contains optional information about the original source code. - // You may safely remove this entire field whithout harming runtime + // You may safely remove this entire field without harming runtime // functionality of the descriptors -- the information is needed only by // development tools. optional SourceCodeInfo source_code_info = 9; + + // The syntax of the proto file. + // The supported values are "proto2" and "proto3". + optional string syntax = 12; } // Describes a message type. @@ -93,10 +102,34 @@ message DescriptorProto { message ExtensionRange { optional int32 start = 1; optional int32 end = 2; + + optional ExtensionRangeOptions options = 3; } repeated ExtensionRange extension_range = 5; + repeated OneofDescriptorProto oneof_decl = 8; + optional MessageOptions options = 7; + + // Range of reserved tag numbers. Reserved tag numbers may not be used by + // fields or extension ranges in the same message. Reserved ranges may + // not overlap. + message ReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Exclusive. + } + repeated ReservedRange reserved_range = 9; + // Reserved field names, which may not be used by fields in the same message. + // A given name may only be reserved once. + repeated string reserved_name = 10; +} + +message ExtensionRangeOptions { + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; } // Describes a field within a message. @@ -117,7 +150,11 @@ message FieldDescriptorProto { TYPE_FIXED32 = 7; TYPE_BOOL = 8; TYPE_STRING = 9; - TYPE_GROUP = 10; // Tag-delimited aggregate. + // Tag-delimited aggregate. + // Group type is deprecated and not supported in proto3. However, Proto3 + // implementations should still be able to parse the group wire format and + // treat group fields as unknown fields. + TYPE_GROUP = 10; TYPE_MESSAGE = 11; // Length-delimited aggregate. // New in version 2. @@ -135,7 +172,6 @@ message FieldDescriptorProto { LABEL_OPTIONAL = 1; LABEL_REQUIRED = 2; LABEL_REPEATED = 3; - // TODO(sanjay): Should we add LABEL_MAP? }; optional string name = 1; @@ -143,7 +179,7 @@ message FieldDescriptorProto { optional Label label = 4; // If type_name is set, this need not be set. If both this and type_name - // are set, this must be either TYPE_ENUM or TYPE_MESSAGE. + // are set, this must be one of TYPE_ENUM, TYPE_MESSAGE or TYPE_GROUP. optional Type type = 5; // For message and enum types, this is the name of the type. If the name @@ -164,9 +200,25 @@ message FieldDescriptorProto { // TODO(kenton): Base-64 encode? optional string default_value = 7; + // If set, gives the index of a oneof in the containing type's oneof_decl + // list. This field is a member of that oneof. + optional int32 oneof_index = 9; + + // JSON name of this field. The value is set by protocol compiler. If the + // user has set a "json_name" option on this field, that option's value + // will be used. Otherwise, it's deduced from the field's name by converting + // it to camelCase. + optional string json_name = 10; + optional FieldOptions options = 8; } +// Describes a oneof. +message OneofDescriptorProto { + optional string name = 1; + optional OneofOptions options = 2; +} + // Describes an enum type. message EnumDescriptorProto { optional string name = 1; @@ -174,6 +226,26 @@ message EnumDescriptorProto { repeated EnumValueDescriptorProto value = 2; optional EnumOptions options = 3; + + // Range of reserved numeric values. Reserved values may not be used by + // entries in the same enum. Reserved ranges may not overlap. + // + // Note that this is distinct from DescriptorProto.ReservedRange in that it + // is inclusive such that it can appropriately represent the entire int32 + // domain. + message EnumReservedRange { + optional int32 start = 1; // Inclusive. + optional int32 end = 2; // Inclusive. + } + + // Range of reserved numeric values. Reserved numeric values may not be used + // by enum values in the same enum declaration. Reserved ranges may not + // overlap. + repeated EnumReservedRange reserved_range = 4; + + // Reserved enum value names, which may not be reused. A given name may only + // be reserved once. + repeated string reserved_name = 5; } // Describes a value within an enum. @@ -202,6 +274,11 @@ message MethodDescriptorProto { optional string output_type = 3; optional MethodOptions options = 4; + + // Identifies if client streams multiple client messages + optional bool client_streaming = 5 [default=false]; + // Identifies if server streams multiple server messages + optional bool server_streaming = 6 [default=false]; } @@ -228,12 +305,12 @@ message MethodDescriptorProto { // * For options which will be published and used publicly by multiple // independent entities, e-mail protobuf-global-extension-registry@google.com // to reserve extension numbers. Simply provide your project name (e.g. -// Object-C plugin) and your porject website (if available) -- there's no need -// to explain how you intend to use them. Usually you only need one extension -// number. You can declare multiple options with only one extension number by -// putting them in a sub-message. See the Custom Options section of the docs -// for examples: -// http://code.google.com/apis/protocolbuffers/docs/proto.html#options +// Objective-C plugin) and your project website (if available) -- there's no +// need to explain how you intend to use them. Usually you only need one +// extension number. You can declare multiple options with only one extension +// number by putting them in a sub-message. See the Custom Options section of +// the docs for examples: +// https://developers.google.com/protocol-buffers/docs/proto#options // If this turns out to be popular, a web service will be set up // to automatically assign option numbers. @@ -262,11 +339,17 @@ message FileOptions { // top-level extensions defined in the file. optional bool java_multiple_files = 10 [default=false]; - // If set true, then the Java code generator will generate equals() and - // hashCode() methods for all messages defined in the .proto file. This is - // purely a speed optimization, as the AbstractMessage base class includes - // reflection-based implementations of these methods. - optional bool java_generate_equals_and_hash = 20 [default=false]; + // This option does nothing. + optional bool java_generate_equals_and_hash = 20 [deprecated=true]; + + // If set true, then the Java2 code generator will generate code that + // throws an exception whenever an attempt is made to assign a non-UTF-8 + // byte sequence to a string field. + // Message reflection will do the same. + // However, an extension field still accepts non-UTF-8 byte sequences. + // This option has no effect on when used with the lite runtime. + optional bool java_string_check_utf8 = 27 [default=false]; + // Generated classes can be optimized for speed or code size. enum OptimizeMode { @@ -278,7 +361,10 @@ message FileOptions { optional OptimizeMode optimize_for = 9 [default=SPEED]; // Sets the Go package where structs generated from this .proto will be - // placed. There is no default. + // placed. If omitted, the Go package will be derived from the following: + // - The basename of the package import path, if provided. + // - Otherwise, the package statement in the .proto file, if present. + // - Otherwise, the basename of the .proto file, without extension. optional string go_package = 11; @@ -287,7 +373,7 @@ message FileOptions { // are not specific to any particular RPC system. They are generated by the // main code generators in each language (without additional plugins). // Generic services were the only kind of service generation supported by - // early versions of proto2. + // early versions of google.protobuf. // // Generic services are now considered deprecated in favor of using plugins // that generate code specific to your particular RPC system. Therefore, @@ -296,12 +382,50 @@ message FileOptions { optional bool cc_generic_services = 16 [default=false]; optional bool java_generic_services = 17 [default=false]; optional bool py_generic_services = 18 [default=false]; + optional bool php_generic_services = 42 [default=false]; - // The parser stores options it doesn't recognize here. See above. + // Is this file deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for everything in the file, or it will be completely ignored; in the very + // least, this is a formalization for deprecating files. + optional bool deprecated = 23 [default=false]; + + // Enables the use of arenas for the proto messages in this file. This applies + // only to generated classes for C++. + optional bool cc_enable_arenas = 31 [default=false]; + + + // Sets the objective c class prefix which is prepended to all objective c + // generated classes from this .proto. There is no default. + optional string objc_class_prefix = 36; + + // Namespace for generated classes; defaults to the package. + optional string csharp_namespace = 37; + + // By default Swift generators will take the proto package and CamelCase it + // replacing '.' with underscore and use that to prefix the types/symbols + // defined. When this options is provided, they will use this value instead + // to prefix the types/symbols defined. + optional string swift_prefix = 39; + + // Sets the php class prefix which is prepended to all php generated classes + // from this .proto. Default is empty. + optional string php_class_prefix = 40; + + // Use this option to change the namespace of php generated classes. Default + // is empty. When this option is empty, the package name will be used for + // determining the namespace. + optional string php_namespace = 41; + + // The parser stores options it doesn't recognize here. + // See the documentation for the "Options" section above. repeated UninterpretedOption uninterpreted_option = 999; - // Clients can define custom options in extensions of this message. See above. + // Clients can define custom options in extensions of this message. + // See the documentation for the "Options" section above. extensions 1000 to max; + + reserved 38; } message MessageOptions { @@ -330,6 +454,38 @@ message MessageOptions { // from proto1 easier; new code should avoid fields named "descriptor". optional bool no_standard_descriptor_accessor = 2 [default=false]; + // Is this message deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the message, or it will be completely ignored; in the very least, + // this is a formalization for deprecating messages. + optional bool deprecated = 3 [default=false]; + + // Whether the message is an automatically generated map entry type for the + // maps field. + // + // For maps fields: + // map<KeyType, ValueType> map_field = 1; + // The parsed descriptor looks like: + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; + // + // Implementations may choose not to generate the map_entry=true message, but + // use a native map in the target language to hold the keys and values. + // The reflection APIs in such implementions still need to work as + // if the field is a repeated message field. + // + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + optional bool map_entry = 7; + + reserved 8; // javalite_serializable + reserved 9; // javanano_as_lite + // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -354,10 +510,32 @@ message FieldOptions { // The packed option can be enabled for repeated primitive fields to enable // a more efficient representation on the wire. Rather than repeatedly // writing the tag and type for each element, the entire array is encoded as - // a single length-delimited blob. + // a single length-delimited blob. In proto3, only explicit setting it to + // false will avoid using packed encoding. optional bool packed = 2; - + // The jstype option determines the JavaScript type used for values of the + // field. The option is permitted only for 64 bit integral and fixed types + // (int64, uint64, sint64, fixed64, sfixed64). A field with jstype JS_STRING + // is represented as JavaScript string, which avoids loss of precision that + // can happen when a large value is converted to a floating point JavaScript. + // Specifying JS_NUMBER for the jstype causes the generated JavaScript code to + // use the JavaScript "number" type. The behavior of the default option + // JS_NORMAL is implementation dependent. + // + // This option is an enum to permit additional types to be added, e.g. + // goog.math.Integer. + optional JSType jstype = 6 [default = JS_NORMAL]; + enum JSType { + // Use the default type. + JS_NORMAL = 0; + + // Use JavaScript strings. + JS_STRING = 1; + + // Use JavaScript numbers. + JS_NUMBER = 2; + } // Should this field be parsed lazily? Lazy applies only to message-type // fields. It means that when the outer message is initially parsed, the @@ -378,7 +556,7 @@ message FieldOptions { // // // Note that implementations may choose not to check required fields within - // a lazy sub-message. That is, calling IsInitialized() on the outher message + // a lazy sub-message. That is, calling IsInitialized() on the outer message // may return true even if the inner message has missing required fields. // This is necessary because otherwise the inner message would have to be // parsed in order to perform the check, defeating the purpose of lazy @@ -395,23 +573,20 @@ message FieldOptions { // is a formalization for deprecating fields. optional bool deprecated = 3 [default=false]; - // EXPERIMENTAL. DO NOT USE. - // For "map" fields, the name of the field in the enclosed type that - // is the key for this map. For example, suppose we have: - // message Item { - // required string name = 1; - // required string value = 2; - // } - // message Config { - // repeated Item items = 1 [experimental_map_key="name"]; - // } - // In this situation, the map key for Item will be set to "name". - // TODO: Fully-implement this, then remove the "experimental_" prefix. - optional string experimental_map_key = 9; - // For Google-internal migration only. Do not use. optional bool weak = 10 [default=false]; + + // The parser stores options it doesn't recognize here. See above. + repeated UninterpretedOption uninterpreted_option = 999; + + // Clients can define custom options in extensions of this message. See above. + extensions 1000 to max; + + reserved 4; // removed jtype +} + +message OneofOptions { // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -421,9 +596,17 @@ message FieldOptions { message EnumOptions { - // Set this option to false to disallow mapping different tag names to a same + // Set this option to true to allow mapping different tag names to the same // value. - optional bool allow_alias = 2 [default=true]; + optional bool allow_alias = 2; + + // Is this enum deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum, or it will be completely ignored; in the very least, this + // is a formalization for deprecating enums. + optional bool deprecated = 3 [default=false]; + + reserved 5; // javanano_as_lite // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -433,6 +616,12 @@ message EnumOptions { } message EnumValueOptions { + // Is this enum value deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the enum value, or it will be completely ignored; in the very least, + // this is a formalization for deprecating enum values. + optional bool deprecated = 1 [default=false]; + // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -447,6 +636,12 @@ message ServiceOptions { // we were already using them long before we decided to release Protocol // Buffers. + // Is this service deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the service, or it will be completely ignored; in the very least, + // this is a formalization for deprecating services. + optional bool deprecated = 33 [default=false]; + // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -461,6 +656,23 @@ message MethodOptions { // we were already using them long before we decided to release Protocol // Buffers. + // Is this method deprecated? + // Depending on the target platform, this can emit Deprecated annotations + // for the method, or it will be completely ignored; in the very least, + // this is a formalization for deprecating methods. + optional bool deprecated = 33 [default=false]; + + // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, + // or neither? HTTP based RPC implementation may choose GET verb for safe + // methods, and PUT verb for idempotent methods instead of the default POST. + enum IdempotencyLevel { + IDEMPOTENCY_UNKNOWN = 0; + NO_SIDE_EFFECTS = 1; // implies idempotent + IDEMPOTENT = 2; // idempotent, but may have side effects + } + optional IdempotencyLevel idempotency_level = + 34 [default=IDEMPOTENCY_UNKNOWN]; + // The parser stores options it doesn't recognize here. See above. repeated UninterpretedOption uninterpreted_option = 999; @@ -587,6 +799,11 @@ message SourceCodeInfo { // A series of line comments appearing on consecutive lines, with no other // tokens appearing on those lines, will be treated as a single comment. // + // leading_detached_comments will keep paragraphs of comments that appear + // before (but not connected to) the current element. Each paragraph, + // separated by empty lines, will be one comment element in the repeated + // field. + // // Only the comment content is provided; comment markers (e.g. //) are // stripped out. For block comments, leading whitespace and an asterisk // will be stripped from the beginning of each line other than the first. @@ -607,6 +824,12 @@ message SourceCodeInfo { // // Another line attached to qux. // optional double qux = 4; // + // // Detached comment for corge. This is not leading or trailing comments + // // to qux or corge because there are blank lines separating it from + // // both. + // + // // Detached comment for corge paragraph 2. + // // optional string corge = 5; // /* Block comment attached // * to corge. Leading asterisks @@ -614,7 +837,36 @@ message SourceCodeInfo { // /* Block comment attached to // * grault. */ // optional int32 grault = 6; + // + // // ignored detached comments. optional string leading_comments = 3; optional string trailing_comments = 4; + repeated string leading_detached_comments = 6; + } +} + +// Describes the relationship between generated code and its original source +// file. A GeneratedCodeInfo message is associated with only one generated +// source file, but may contain references to different source .proto files. +message GeneratedCodeInfo { + // An Annotation connects some span of text in generated code to an element + // of its generating .proto file. + repeated Annotation annotation = 1; + message Annotation { + // Identifies the element in the original source .proto file. This field + // is formatted the same as SourceCodeInfo.Location.path. + repeated int32 path = 1 [packed=true]; + + // Identifies the filesystem path to the original source .proto. + optional string source_file = 2; + + // Identifies the starting offset in bytes in the generated code + // that relates to the identified object. + optional int32 begin = 3; + + // Identifies the ending offset in bytes in the generated code that + // relates to the identified offset. The end offset should be one past + // the last relevant byte (so the length of the text = end - begin). + optional int32 end = 4; } } diff --git a/generator/proto/nanopb.proto b/generator/proto/nanopb.proto index 2be2f80..0c05a2b 100644 --- a/generator/proto/nanopb.proto +++ b/generator/proto/nanopb.proto @@ -5,6 +5,7 @@ // These are used by nanopb to generate statically allocable structures // for memory-limited environments. +syntax = "proto2"; import "google/protobuf/descriptor.proto"; option java_package = "fi.kapsi.koti.jpa.nanopb"; @@ -15,6 +16,15 @@ enum FieldType { FT_POINTER = 4; // Always generate a dynamically allocated field. FT_STATIC = 2; // Generate a static field or raise an exception if not possible. FT_IGNORE = 3; // Ignore the field completely. + FT_INLINE = 5; // Legacy option, use the separate 'fixed_length' option instead +} + +enum IntSize { + IS_DEFAULT = 0; // Default, 32/64bit based on type in .proto + IS_8 = 8; + IS_16 = 16; + IS_32 = 32; + IS_64 = 64; } // This is the inner options message, which basically defines options for @@ -22,11 +32,20 @@ enum FieldType { // fields. message NanoPBOptions { // Allocated size for 'bytes' and 'string' fields. + // For string fields, this should include the space for null terminator. optional int32 max_size = 1; + // Maximum length for 'string' fields. Setting this is equivalent + // to setting max_size to a value of length+1. + optional int32 max_length = 14; + // Allocated number of entries in arrays ('repeated' fields) optional int32 max_count = 2; + // Size of integer fields. Can save some memory if you don't need + // full 32 bits for the value. + optional IntSize int_size = 7 [default = IS_DEFAULT]; + // Force type of field (callback or static allocation) optional FieldType type = 3 [default = FT_DEFAULT]; @@ -37,6 +56,33 @@ message NanoPBOptions { // Note: this cannot be used on CPUs that break on unaligned // accesses to variables. optional bool packed_struct = 5 [default = false]; + + // Add 'packed' attribute to generated enums. + optional bool packed_enum = 10 [default = false]; + + // Skip this message + optional bool skip_message = 6 [default = false]; + + // Generate oneof fields as normal optional fields instead of union. + optional bool no_unions = 8 [default = false]; + + // integer type tag for a message + optional uint32 msgid = 9; + + // decode oneof as anonymous union + optional bool anonymous_oneof = 11 [default = false]; + + // Proto3 singular field does not generate a "has_" flag + optional bool proto3 = 12 [default = false]; + + // Generate an enum->string mapping function (can take up lots of space). + optional bool enum_to_string = 13 [default = false]; + + // Generate bytes arrays with fixed length + optional bool fixed_length = 15 [default = false]; + + // Generate repeated field with fixed count + optional bool fixed_count = 16 [default = false]; } // Extensions to protoc 'Descriptor' type in order to define options diff --git a/generator/proto/nanopb_pb2.py b/generator/proto/nanopb_pb2.py deleted file mode 100644 index 7291eae..0000000 --- a/generator/proto/nanopb_pb2.py +++ /dev/null @@ -1,162 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! - -from google.protobuf import descriptor -from google.protobuf import message -from google.protobuf import reflection -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - - -DESCRIPTOR = descriptor.FileDescriptor( - name='nanopb.proto', - package='', - serialized_pb='\n\x0cnanopb.proto\x1a google/protobuf/descriptor.proto\"\x92\x01\n\rNanoPBOptions\x12\x10\n\x08max_size\x18\x01 \x01(\x05\x12\x11\n\tmax_count\x18\x02 \x01(\x05\x12$\n\x04type\x18\x03 \x01(\x0e\x32\n.FieldType:\nFT_DEFAULT\x12\x18\n\nlong_names\x18\x04 \x01(\x08:\x04true\x12\x1c\n\rpacked_struct\x18\x05 \x01(\x08:\x05\x66\x61lse*Z\n\tFieldType\x12\x0e\n\nFT_DEFAULT\x10\x00\x12\x0f\n\x0b\x46T_CALLBACK\x10\x01\x12\x0e\n\nFT_POINTER\x10\x04\x12\r\n\tFT_STATIC\x10\x02\x12\r\n\tFT_IGNORE\x10\x03:E\n\x0enanopb_fileopt\x12\x1c.google.protobuf.FileOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:G\n\rnanopb_msgopt\x12\x1f.google.protobuf.MessageOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:E\n\x0enanopb_enumopt\x12\x1c.google.protobuf.EnumOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptions:>\n\x06nanopb\x12\x1d.google.protobuf.FieldOptions\x18\xf2\x07 \x01(\x0b\x32\x0e.NanoPBOptionsB\x1a\n\x18\x66i.kapsi.koti.jpa.nanopb') - -_FIELDTYPE = descriptor.EnumDescriptor( - name='FieldType', - full_name='FieldType', - filename=None, - file=DESCRIPTOR, - values=[ - descriptor.EnumValueDescriptor( - name='FT_DEFAULT', index=0, number=0, - options=None, - type=None), - descriptor.EnumValueDescriptor( - name='FT_CALLBACK', index=1, number=1, - options=None, - type=None), - descriptor.EnumValueDescriptor( - name='FT_POINTER', index=2, number=4, - options=None, - type=None), - descriptor.EnumValueDescriptor( - name='FT_STATIC', index=3, number=2, - options=None, - type=None), - descriptor.EnumValueDescriptor( - name='FT_IGNORE', index=4, number=3, - options=None, - type=None), - ], - containing_type=None, - options=None, - serialized_start=199, - serialized_end=289, -) - - -FT_DEFAULT = 0 -FT_CALLBACK = 1 -FT_POINTER = 4 -FT_STATIC = 2 -FT_IGNORE = 3 - -NANOPB_FILEOPT_FIELD_NUMBER = 1010 -nanopb_fileopt = descriptor.FieldDescriptor( - name='nanopb_fileopt', full_name='nanopb_fileopt', index=0, - number=1010, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - options=None) -NANOPB_MSGOPT_FIELD_NUMBER = 1010 -nanopb_msgopt = descriptor.FieldDescriptor( - name='nanopb_msgopt', full_name='nanopb_msgopt', index=1, - number=1010, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - options=None) -NANOPB_ENUMOPT_FIELD_NUMBER = 1010 -nanopb_enumopt = descriptor.FieldDescriptor( - name='nanopb_enumopt', full_name='nanopb_enumopt', index=2, - number=1010, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - options=None) -NANOPB_FIELD_NUMBER = 1010 -nanopb = descriptor.FieldDescriptor( - name='nanopb', full_name='nanopb', index=3, - number=1010, type=11, cpp_type=10, label=1, - has_default_value=False, default_value=None, - message_type=None, enum_type=None, containing_type=None, - is_extension=True, extension_scope=None, - options=None) - - -_NANOPBOPTIONS = descriptor.Descriptor( - name='NanoPBOptions', - full_name='NanoPBOptions', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='max_size', full_name='NanoPBOptions.max_size', index=0, - number=1, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='max_count', full_name='NanoPBOptions.max_count', index=1, - number=2, type=5, cpp_type=1, label=1, - has_default_value=False, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='type', full_name='NanoPBOptions.type', index=2, - number=3, type=14, cpp_type=8, label=1, - has_default_value=True, default_value=0, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='long_names', full_name='NanoPBOptions.long_names', index=3, - number=4, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=True, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='packed_struct', full_name='NanoPBOptions.packed_struct', index=4, - number=5, type=8, cpp_type=7, label=1, - has_default_value=True, default_value=False, - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=51, - serialized_end=197, -) - -import google.protobuf.descriptor_pb2 - -_NANOPBOPTIONS.fields_by_name['type'].enum_type = _FIELDTYPE - -class NanoPBOptions(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _NANOPBOPTIONS - - # @@protoc_insertion_point(class_scope:NanoPBOptions) - -nanopb_fileopt.message_type = _NANOPBOPTIONS -google.protobuf.descriptor_pb2.FileOptions.RegisterExtension(nanopb_fileopt) -nanopb_msgopt.message_type = _NANOPBOPTIONS -google.protobuf.descriptor_pb2.MessageOptions.RegisterExtension(nanopb_msgopt) -nanopb_enumopt.message_type = _NANOPBOPTIONS -google.protobuf.descriptor_pb2.EnumOptions.RegisterExtension(nanopb_enumopt) -nanopb.message_type = _NANOPBOPTIONS -google.protobuf.descriptor_pb2.FieldOptions.RegisterExtension(nanopb) -# @@protoc_insertion_point(module_scope) diff --git a/generator/proto/plugin.proto b/generator/proto/plugin.proto index 651ed10..e627289 100644 --- a/generator/proto/plugin.proto +++ b/generator/proto/plugin.proto @@ -1,6 +1,6 @@ // Protocol Buffers - Google's data interchange format // Copyright 2008 Google Inc. All rights reserved. -// http://code.google.com/p/protobuf/ +// https://developers.google.com/protocol-buffers/ // // Redistribution and use in source and binary forms, with or without // modification, are permitted provided that the following conditions are @@ -44,7 +44,10 @@ // plugin should be named "protoc-gen-$NAME", and will then be used when the // flag "--${NAME}_out" is passed to protoc. +syntax = "proto2"; package google.protobuf.compiler; +option java_package = "com.google.protobuf.compiler"; +option java_outer_classname = "PluginProtos"; import "google/protobuf/descriptor.proto"; diff --git a/generator/proto/plugin_pb2.py b/generator/proto/plugin_pb2.py deleted file mode 100644 index a0a6bb7..0000000 --- a/generator/proto/plugin_pb2.py +++ /dev/null @@ -1,159 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! - -from google.protobuf import descriptor -from google.protobuf import message -from google.protobuf import reflection -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - - -DESCRIPTOR = descriptor.FileDescriptor( - name='plugin.proto', - package='google.protobuf.compiler', - serialized_pb='\n\x0cplugin.proto\x12\x18google.protobuf.compiler\x1a google/protobuf/descriptor.proto\"}\n\x14\x43odeGeneratorRequest\x12\x18\n\x10\x66ile_to_generate\x18\x01 \x03(\t\x12\x11\n\tparameter\x18\x02 \x01(\t\x12\x38\n\nproto_file\x18\x0f \x03(\x0b\x32$.google.protobuf.FileDescriptorProto\"\xaa\x01\n\x15\x43odeGeneratorResponse\x12\r\n\x05\x65rror\x18\x01 \x01(\t\x12\x42\n\x04\x66ile\x18\x0f \x03(\x0b\x32\x34.google.protobuf.compiler.CodeGeneratorResponse.File\x1a>\n\x04\x46ile\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x17\n\x0finsertion_point\x18\x02 \x01(\t\x12\x0f\n\x07\x63ontent\x18\x0f \x01(\t') - - - - -_CODEGENERATORREQUEST = descriptor.Descriptor( - name='CodeGeneratorRequest', - full_name='google.protobuf.compiler.CodeGeneratorRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='file_to_generate', full_name='google.protobuf.compiler.CodeGeneratorRequest.file_to_generate', index=0, - number=1, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='parameter', full_name='google.protobuf.compiler.CodeGeneratorRequest.parameter', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='proto_file', full_name='google.protobuf.compiler.CodeGeneratorRequest.proto_file', index=2, - number=15, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=76, - serialized_end=201, -) - - -_CODEGENERATORRESPONSE_FILE = descriptor.Descriptor( - name='File', - full_name='google.protobuf.compiler.CodeGeneratorResponse.File', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='name', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.name', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='insertion_point', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.insertion_point', index=1, - number=2, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='content', full_name='google.protobuf.compiler.CodeGeneratorResponse.File.content', index=2, - number=15, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=312, - serialized_end=374, -) - -_CODEGENERATORRESPONSE = descriptor.Descriptor( - name='CodeGeneratorResponse', - full_name='google.protobuf.compiler.CodeGeneratorResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - descriptor.FieldDescriptor( - name='error', full_name='google.protobuf.compiler.CodeGeneratorResponse.error', index=0, - number=1, type=9, cpp_type=9, label=1, - has_default_value=False, default_value=unicode("", "utf-8"), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - descriptor.FieldDescriptor( - name='file', full_name='google.protobuf.compiler.CodeGeneratorResponse.file', index=1, - number=15, type=11, cpp_type=10, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[_CODEGENERATORRESPONSE_FILE, ], - enum_types=[ - ], - options=None, - is_extendable=False, - extension_ranges=[], - serialized_start=204, - serialized_end=374, -) - -import google.protobuf.descriptor_pb2 - -_CODEGENERATORREQUEST.fields_by_name['proto_file'].message_type = google.protobuf.descriptor_pb2._FILEDESCRIPTORPROTO -_CODEGENERATORRESPONSE_FILE.containing_type = _CODEGENERATORRESPONSE; -_CODEGENERATORRESPONSE.fields_by_name['file'].message_type = _CODEGENERATORRESPONSE_FILE - -class CodeGeneratorRequest(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _CODEGENERATORREQUEST - - # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorRequest) - -class CodeGeneratorResponse(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - - class File(message.Message): - __metaclass__ = reflection.GeneratedProtocolMessageType - DESCRIPTOR = _CODEGENERATORRESPONSE_FILE - - # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse.File) - DESCRIPTOR = _CODEGENERATORRESPONSE - - # @@protoc_insertion_point(class_scope:google.protobuf.compiler.CodeGeneratorResponse) - -# @@protoc_insertion_point(module_scope) diff --git a/generator/protoc-gen-nanopb b/generator/protoc-gen-nanopb index 2de5621..471a620 100755 --- a/generator/protoc-gen-nanopb +++ b/generator/protoc-gen-nanopb @@ -3,11 +3,11 @@ # This file is used to invoke nanopb_generator.py as a plugin # to protoc on Linux and other *nix-style systems. # Use it like this: -# protoc --plugin=nanopb=..../protoc-gen-nanopb --nanopb_out=dir foo.proto +# protoc --plugin=protoc-gen-nanopb=..../protoc-gen-nanopb --nanopb_out=dir foo.proto # # Note that if you use the binary package of nanopb, the protoc # path is already set up properly and there is no need to give # --plugin= on the command line. MYPATH=$(dirname "$0") -exec python "$MYPATH/nanopb_generator.py" --protoc-plugin +exec "$MYPATH/nanopb_generator.py" --protoc-plugin diff --git a/generator/protoc-gen-nanopb.bat b/generator/protoc-gen-nanopb.bat index 7624984..e6cf187 100644 --- a/generator/protoc-gen-nanopb.bat +++ b/generator/protoc-gen-nanopb.bat @@ -2,7 +2,7 @@ :: This file is used to invoke nanopb_generator.py as a plugin :: to protoc on Windows. :: Use it like this: -:: protoc --plugin=nanopb=..../protoc-gen-nanopb.bat --nanopb_out=dir foo.proto +:: protoc --plugin=protoc-gen-nanopb=..../protoc-gen-nanopb.bat --nanopb_out=dir foo.proto :: :: Note that if you use the binary package of nanopb, the protoc :: path is already set up properly and there is no need to give diff --git a/library.json b/library.json new file mode 100644 index 0000000..b070e54 --- /dev/null +++ b/library.json @@ -0,0 +1,26 @@ +{ + "name": "Nanopb", + "version": "0.3.9.1", + "keywords": "protocol buffers, protobuf, google", + "description": "Nanopb is a plain-C implementation of Google's Protocol Buffers data format. It is targeted at 32 bit microcontrollers, but is also fit for other embedded systems with tight (2-10 kB ROM, <1 kB RAM) memory constraints.", + "repository": { + "type": "git", + "url": "https://github.com/nanopb/nanopb.git" + }, + "authors": [{ + "name": "Petteri Aimonen", + "email": "jpa@nanopb.mail.kapsi.fi", + "url": "http://koti.kapsi.fi/jpa/nanopb/" + }], + "export": { + "include": [ + "*.c", + "*.cpp", + "*.h", + "examples" + ] + }, + "examples": "examples/*/*.c", + "frameworks": "*", + "platforms": "*" +} diff --git a/patches/pb.h.patch b/patches/pb.h.patch new file mode 100644 index 0000000..f269b7d --- /dev/null +++ b/patches/pb.h.patch @@ -0,0 +1,27 @@ +diff --git a/pb.h b/pb.h +index 174a84b..52d24de 100644 +--- a/pb.h ++++ b/pb.h +@@ -122,6 +122,11 @@ + #define PB_STATIC_ASSERT_MSG(MSG, LINE, COUNTER) PB_STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) + #define PB_STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) pb_static_assertion_##MSG##LINE##COUNTER + #endif ++#ifndef STATIC_ASSERT ++#define STATIC_ASSERT PB_STATIC_ASSERT ++#define STATIC_ASSERT_MSG PB_STATIC_ASSERT_MSG ++#define STATIC_ASSERT_MSG_ PB_STATIC_ASSERT_MSG_ ++#endif + #else + #define PB_STATIC_ASSERT(COND,MSG) + #endif +@@ -526,6 +531,7 @@ struct pb_extension_s { + PB_ ## rules ## _ ## allocation(tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) ++#define PB_FIELD2 PB_FIELD + + /* Field description for repeated static fixed count fields.*/ + #define PB_REPEATED_FIXED_COUNT(tag, type, placement, message, field, prevfield, ptr) \ +-- +2.18.0.345.g5c9ce644c3-goog + @@ -2,8 +2,8 @@ * stuff. For the high-level interface, see pb_encode.h and pb_decode.h. */ -#ifndef _PB_H_ -#define _PB_H_ +#ifndef PB_H_INCLUDED +#define PB_H_INCLUDED /***************************************************************** * Nanopb compilation time options. You can change these here by * @@ -13,9 +13,9 @@ /* Enable support for dynamically allocated fields */ /* #define PB_ENABLE_MALLOC 1 */ -/* Define this if your CPU architecture is big endian, i.e. it - * stores the most-significant byte first. */ -/* #define __BIG_ENDIAN__ 1 */ +/* Define this if your CPU / compiler combination does not support + * unaligned memory access to packed structures. */ +/* #define PB_NO_PACKED_STRUCTS 1 */ /* Increase the number of required fields that are tracked. * A compiler warning will tell if you need this. */ @@ -46,12 +46,12 @@ /* Version of the nanopb library. Just in case you want to check it in * your own program. */ -#define NANOPB_VERSION nanopb-0.2.8-dev +#define NANOPB_VERSION nanopb-0.3.9.1 /* Include all the system headers needed by nanopb. You will need the * definitions of the following: * - strlen, memcpy, memset functions - * - [u]int8_t, [u]int16_t, [u]int32_t, [u]int64_t + * - [u]int_least8_t, uint_fast8_t, [u]int_least16_t, [u]int32_t, [u]int64_t * - size_t * - bool * @@ -75,13 +75,18 @@ /* Macro for defining packed structures (compiler dependent). * This just reduces memory requirements, but is not required. */ -#if defined(__GNUC__) || defined(__clang__) +#if defined(PB_NO_PACKED_STRUCTS) + /* Disable struct packing */ +# define PB_PACKED_STRUCT_START +# define PB_PACKED_STRUCT_END +# define pb_packed +#elif defined(__GNUC__) || defined(__clang__) /* For GCC and clang */ # define PB_PACKED_STRUCT_START # define PB_PACKED_STRUCT_END # define pb_packed __attribute__((packed)) -#elif defined(__ICCARM__) - /* For IAR ARM compiler */ +#elif defined(__ICCARM__) || defined(__CC_ARM) + /* For IAR ARM and Keil MDK-ARM compilers */ # define PB_PACKED_STRUCT_START _Pragma("pack(push, 1)") # define PB_PACKED_STRUCT_END _Pragma("pack(pop)") # define pb_packed @@ -98,23 +103,32 @@ #endif /* Handly macro for suppressing unreferenced-parameter compiler warnings. */ -#ifndef UNUSED -#define UNUSED(x) (void)(x) +#ifndef PB_UNUSED +#define PB_UNUSED(x) (void)(x) #endif /* Compile-time assertion, used for checking compatible compilation options. - * If this does not work properly on your compiler, use #define STATIC_ASSERT - * to disable it. + * If this does not work properly on your compiler, use + * #define PB_NO_STATIC_ASSERT to disable it. * * But before doing that, check carefully the error message / place where it * comes from to see if the error has a real cause. Unfortunately the error * message is not always very clear to read, but you can see the reason better - * in the place where the STATIC_ASSERT macro was called. + * in the place where the PB_STATIC_ASSERT macro was called. */ +#ifndef PB_NO_STATIC_ASSERT +#ifndef PB_STATIC_ASSERT +#define PB_STATIC_ASSERT(COND,MSG) typedef char PB_STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND)?1:-1]; +#define PB_STATIC_ASSERT_MSG(MSG, LINE, COUNTER) PB_STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) +#define PB_STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) pb_static_assertion_##MSG##LINE##COUNTER +#endif #ifndef STATIC_ASSERT -#define STATIC_ASSERT(COND,MSG) typedef char STATIC_ASSERT_MSG(MSG, __LINE__, __COUNTER__)[(COND)?1:-1]; -#define STATIC_ASSERT_MSG(MSG, LINE, COUNTER) STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) -#define STATIC_ASSERT_MSG_(MSG, LINE, COUNTER) static_assertion_##MSG##LINE##COUNTER +#define STATIC_ASSERT PB_STATIC_ASSERT +#define STATIC_ASSERT_MSG PB_STATIC_ASSERT_MSG +#define STATIC_ASSERT_MSG_ PB_STATIC_ASSERT_MSG_ +#endif +#else +#define PB_STATIC_ASSERT(COND,MSG) #endif /* Number of required fields to keep track of. */ @@ -131,7 +145,7 @@ * Most-significant 4 bits specify repeated/required/packed etc. */ -typedef uint8_t pb_type_t; +typedef uint_least8_t pb_type_t; /**** Field data types ****/ @@ -161,8 +175,14 @@ typedef uint8_t pb_type_t; * The field contains a pointer to pb_extension_t */ #define PB_LTYPE_EXTENSION 0x08 +/* Byte array with inline, pre-allocated byffer. + * data_size is the length of the inline, allocated buffer. + * This differs from PB_LTYPE_BYTES by defining the element as + * pb_byte_t[data_size] rather than pb_bytes_array_t. */ +#define PB_LTYPE_FIXED_LENGTH_BYTES 0x09 + /* Number of declared LTYPES */ -#define PB_LTYPES_COUNT 9 +#define PB_LTYPES_COUNT 0x0A #define PB_LTYPE_MASK 0x0F /**** Field repetition rules ****/ @@ -170,6 +190,7 @@ typedef uint8_t pb_type_t; #define PB_HTYPE_REQUIRED 0x00 #define PB_HTYPE_OPTIONAL 0x10 #define PB_HTYPE_REPEATED 0x20 +#define PB_HTYPE_ONEOF 0x30 #define PB_HTYPE_MASK 0x30 /**** Field allocation types ****/ @@ -190,12 +211,19 @@ typedef uint8_t pb_type_t; typedef uint32_t pb_size_t; typedef int32_t pb_ssize_t; #elif defined(PB_FIELD_16BIT) - typedef uint16_t pb_size_t; - typedef int16_t pb_ssize_t; + typedef uint_least16_t pb_size_t; + typedef int_least16_t pb_ssize_t; #else - typedef uint8_t pb_size_t; - typedef int8_t pb_ssize_t; + typedef uint_least8_t pb_size_t; + typedef int_least8_t pb_ssize_t; #endif +#define PB_SIZE_MAX ((pb_size_t)-1) + +/* Data type for storing encoded data and other byte streams. + * This typedef exists to support platforms where uint8_t does not exist. + * You can regard it as equivalent on uint8_t on other platforms. + */ +typedef uint_least8_t pb_byte_t; /* This structure is used in auto-generated constants * to specify struct fields. @@ -206,8 +234,8 @@ typedef uint8_t pb_type_t; * PB_FIELD_32BIT. */ PB_PACKED_STRUCT_START -typedef struct _pb_field_t pb_field_t; -struct _pb_field_t { +typedef struct pb_field_s pb_field_t; +struct pb_field_s { pb_size_t tag; pb_type_t type; pb_size_t data_offset; /* Offset of field data, relative to previous field. */ @@ -223,32 +251,28 @@ struct _pb_field_t { PB_PACKED_STRUCT_END /* Make sure that the standard integer types are of the expected sizes. - * All kinds of things may break otherwise.. atleast all fixed* types. + * Otherwise fixed32/fixed64 fields can break. * * If you get errors here, it probably means that your stdint.h is not * correct for your platform. */ -STATIC_ASSERT(sizeof(int8_t) == 1, INT8_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(uint8_t) == 1, UINT8_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(int16_t) == 2, INT16_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(uint16_t) == 2, UINT16_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(int32_t) == 4, INT32_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(uint32_t) == 4, UINT32_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(int64_t) == 8, INT64_T_WRONG_SIZE) -STATIC_ASSERT(sizeof(uint64_t) == 8, UINT64_T_WRONG_SIZE) +#ifndef PB_WITHOUT_64BIT +PB_STATIC_ASSERT(sizeof(int64_t) == 2 * sizeof(int32_t), INT64_T_WRONG_SIZE) +PB_STATIC_ASSERT(sizeof(uint64_t) == 2 * sizeof(uint32_t), UINT64_T_WRONG_SIZE) +#endif /* This structure is used for 'bytes' arrays. * It has the number of bytes in the beginning, and after that an array. * Note that actual structs used will have a different length of bytes array. */ -#define PB_BYTES_ARRAY_T(n) struct { size_t size; uint8_t bytes[n]; } +#define PB_BYTES_ARRAY_T(n) struct { pb_size_t size; pb_byte_t bytes[n]; } #define PB_BYTES_ARRAY_T_ALLOCSIZE(n) ((size_t)n + offsetof(pb_bytes_array_t, bytes)) -struct _pb_bytes_array_t { - size_t size; - uint8_t bytes[1]; +struct pb_bytes_array_s { + pb_size_t size; + pb_byte_t bytes[1]; }; -typedef struct _pb_bytes_array_t pb_bytes_array_t; +typedef struct pb_bytes_array_s pb_bytes_array_t; /* This structure is used for giving the callback function. * It is stored in the message structure and filled in by the method that @@ -268,10 +292,10 @@ typedef struct _pb_bytes_array_t pb_bytes_array_t; * * The callback can be null if you want to skip a field. */ -typedef struct _pb_istream_t pb_istream_t; -typedef struct _pb_ostream_t pb_ostream_t; -typedef struct _pb_callback_t pb_callback_t; -struct _pb_callback_t { +typedef struct pb_istream_s pb_istream_t; +typedef struct pb_ostream_s pb_ostream_t; +typedef struct pb_callback_s pb_callback_t; +struct pb_callback_s { #ifdef PB_OLD_CALLBACK_STYLE /* Deprecated since nanopb-0.2.1 */ union { @@ -304,9 +328,9 @@ typedef enum { * if you want to catch all unknown fields, you can also create a custom * pb_extension_type_t with your own callback. */ -typedef struct _pb_extension_type_t pb_extension_type_t; -typedef struct _pb_extension_t pb_extension_t; -struct _pb_extension_type_t { +typedef struct pb_extension_type_s pb_extension_type_t; +typedef struct pb_extension_s pb_extension_t; +struct pb_extension_type_s { /* Called for each unknown field in the message. * If you handle the field, read off all of its data and return true. * If you do not handle the field, do not read anything and return true. @@ -328,7 +352,7 @@ struct _pb_extension_type_t { const void *arg; }; -struct _pb_extension_t { +struct pb_extension_s { /* Type describing the extension field. Usually you'll initialize * this to a pointer to the automatically generated structure. */ const pb_extension_type_t *type; @@ -358,6 +382,9 @@ struct _pb_extension_t { # endif #endif +/* This is used to inform about need to regenerate .pb.h/.pb.c files. */ +#define PB_PROTO_HEADER_VERSION 30 + /* These macros are used to declare pb_field_t's in the constant array. */ /* Size of a structure member, in bytes. */ #define pb_membersize(st, m) (sizeof ((st*)0)->m) @@ -373,6 +400,8 @@ struct _pb_extension_t { #define PB_DATAOFFSET_FIRST(st, m1, m2) (offsetof(st, m1)) /* data_offset for subsequent fields */ #define PB_DATAOFFSET_OTHER(st, m1, m2) (offsetof(st, m1) - offsetof(st, m2) - pb_membersize(st, m2)) +/* data offset for subsequent fields inside an union (oneof) */ +#define PB_DATAOFFSET_UNION(st, m1, m2) (PB_SIZE_MAX) /* Choose first/other based on m1 == m2 (deprecated, remains for backwards compatibility) */ #define PB_DATAOFFSET_CHOOSE(st, m1, m2) (int)(offsetof(st, m1) == offsetof(st, m2) \ ? PB_DATAOFFSET_FIRST(st, m1, m2) \ @@ -393,6 +422,10 @@ struct _pb_extension_t { pb_delta(st, has_ ## m, m), \ pb_membersize(st, m), 0, ptr} +#define PB_SINGULAR_STATIC(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} + /* Repeated fields have a _count field and also the maximum number of entries. */ #define PB_REPEATED_STATIC(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | ltype, \ @@ -411,6 +444,11 @@ struct _pb_extension_t { {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \ fd, 0, pb_membersize(st, m[0]), 0, ptr} +/* Same as optional fields*/ +#define PB_SINGULAR_POINTER(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m[0]), 0, ptr} + /* Repeated fields have a _count field and a pointer to array of pointers */ #define PB_REPEATED_POINTER(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_POINTER | PB_HTYPE_REPEATED | ltype, \ @@ -425,95 +463,137 @@ struct _pb_extension_t { #define PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ fd, 0, pb_membersize(st, m), 0, ptr} + +#define PB_SINGULAR_CALLBACK(tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ + fd, 0, pb_membersize(st, m), 0, ptr} #define PB_REPEATED_CALLBACK(tag, st, m, fd, ltype, ptr) \ {tag, PB_ATYPE_CALLBACK | PB_HTYPE_REPEATED | ltype, \ fd, 0, pb_membersize(st, m), 0, ptr} -/* Optional extensions don't have the has_ field, as that would be redundant. */ +/* Optional extensions don't have the has_ field, as that would be redundant. + * Furthermore, the combination of OPTIONAL without has_ field is used + * for indicating proto3 style fields. Extensions exist in proto2 mode only, + * so they should be encoded according to proto2 rules. To avoid the conflict, + * extensions are marked as REQUIRED instead. + */ #define PB_OPTEXT_STATIC(tag, st, m, fd, ltype, ptr) \ - {tag, PB_ATYPE_STATIC | PB_HTYPE_OPTIONAL | ltype, \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REQUIRED | ltype, \ 0, \ 0, \ pb_membersize(st, m), 0, ptr} +#define PB_OPTEXT_POINTER(tag, st, m, fd, ltype, ptr) \ + PB_OPTIONAL_POINTER(tag, st, m, fd, ltype, ptr) + #define PB_OPTEXT_CALLBACK(tag, st, m, fd, ltype, ptr) \ - {tag, PB_ATYPE_CALLBACK | PB_HTYPE_OPTIONAL | ltype, \ - 0, 0, pb_membersize(st, m), 0, ptr} + PB_OPTIONAL_CALLBACK(tag, st, m, fd, ltype, ptr) /* The mapping from protobuf types to LTYPEs is done using these macros. */ -#define PB_LTYPE_MAP_BOOL PB_LTYPE_VARINT -#define PB_LTYPE_MAP_BYTES PB_LTYPE_BYTES -#define PB_LTYPE_MAP_DOUBLE PB_LTYPE_FIXED64 -#define PB_LTYPE_MAP_ENUM PB_LTYPE_VARINT -#define PB_LTYPE_MAP_FIXED32 PB_LTYPE_FIXED32 -#define PB_LTYPE_MAP_FIXED64 PB_LTYPE_FIXED64 -#define PB_LTYPE_MAP_FLOAT PB_LTYPE_FIXED32 -#define PB_LTYPE_MAP_INT32 PB_LTYPE_VARINT -#define PB_LTYPE_MAP_INT64 PB_LTYPE_VARINT -#define PB_LTYPE_MAP_MESSAGE PB_LTYPE_SUBMESSAGE -#define PB_LTYPE_MAP_SFIXED32 PB_LTYPE_FIXED32 -#define PB_LTYPE_MAP_SFIXED64 PB_LTYPE_FIXED64 -#define PB_LTYPE_MAP_SINT32 PB_LTYPE_SVARINT -#define PB_LTYPE_MAP_SINT64 PB_LTYPE_SVARINT -#define PB_LTYPE_MAP_STRING PB_LTYPE_STRING -#define PB_LTYPE_MAP_UINT32 PB_LTYPE_UVARINT -#define PB_LTYPE_MAP_UINT64 PB_LTYPE_UVARINT -#define PB_LTYPE_MAP_EXTENSION PB_LTYPE_EXTENSION +#define PB_LTYPE_MAP_BOOL PB_LTYPE_VARINT +#define PB_LTYPE_MAP_BYTES PB_LTYPE_BYTES +#define PB_LTYPE_MAP_DOUBLE PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_ENUM PB_LTYPE_VARINT +#define PB_LTYPE_MAP_UENUM PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_FIXED32 PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_FIXED64 PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_FLOAT PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_INT32 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_INT64 PB_LTYPE_VARINT +#define PB_LTYPE_MAP_MESSAGE PB_LTYPE_SUBMESSAGE +#define PB_LTYPE_MAP_SFIXED32 PB_LTYPE_FIXED32 +#define PB_LTYPE_MAP_SFIXED64 PB_LTYPE_FIXED64 +#define PB_LTYPE_MAP_SINT32 PB_LTYPE_SVARINT +#define PB_LTYPE_MAP_SINT64 PB_LTYPE_SVARINT +#define PB_LTYPE_MAP_STRING PB_LTYPE_STRING +#define PB_LTYPE_MAP_UINT32 PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_UINT64 PB_LTYPE_UVARINT +#define PB_LTYPE_MAP_EXTENSION PB_LTYPE_EXTENSION +#define PB_LTYPE_MAP_FIXED_LENGTH_BYTES PB_LTYPE_FIXED_LENGTH_BYTES /* This is the actual macro used in field descriptions. * It takes these arguments: * - Field tag number - * - Field type: BOOL, BYTES, DOUBLE, ENUM, FIXED32, FIXED64, + * - Field type: BOOL, BYTES, DOUBLE, ENUM, UENUM, FIXED32, FIXED64, * FLOAT, INT32, INT64, MESSAGE, SFIXED32, SFIXED64 * SINT32, SINT64, STRING, UINT32, UINT64 or EXTENSION * - Field rules: REQUIRED, OPTIONAL or REPEATED - * - Allocation: STATIC or CALLBACK + * - Allocation: STATIC, CALLBACK or POINTER + * - Placement: FIRST or OTHER, depending on if this is the first field in structure. * - Message name * - Field name * - Previous field name (or field name again for first field) * - Pointer to default value or submsg fields. */ -#define PB_FIELD(tag, type, rules, allocation, message, field, prevfield, ptr) \ - PB_ ## rules ## _ ## allocation(tag, message, field, \ - PB_DATAOFFSET_CHOOSE(message, field, prevfield), \ +#define PB_FIELD(tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ + PB_ ## rules ## _ ## allocation(tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ PB_LTYPE_MAP_ ## type, ptr) +#define PB_FIELD2 PB_FIELD -/* This is a new version of the macro used by nanopb generator from - * version 0.2.3 onwards. It avoids the use of a ternary expression in - * the initialization, which confused some compilers. - * - * - Placement: FIRST or OTHER, depending on if this is the first field in structure. - * +/* Field description for repeated static fixed count fields.*/ +#define PB_REPEATED_FIXED_COUNT(tag, type, placement, message, field, prevfield, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_REPEATED | PB_LTYPE_MAP_ ## type, \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ + 0, \ + pb_membersize(message, field[0]), \ + pb_arraysize(message, field), ptr} + +/* Field description for oneof fields. This requires taking into account the + * union name also, that's why a separate set of macros is needed. */ -#define PB_FIELD2(tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ - PB_ ## rules ## _ ## allocation(tag, message, field, \ - PB_DATAOFFSET_ ## placement(message, field, prevfield), \ +#define PB_ONEOF_STATIC(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, u.m), \ + pb_membersize(st, u.m), 0, ptr} + +#define PB_ONEOF_POINTER(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, u.m), \ + pb_membersize(st, u.m[0]), 0, ptr} + +#define PB_ONEOF_FIELD(union_name, tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ + PB_ONEOF_ ## allocation(union_name, tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, union_name.field, prevfield), \ PB_LTYPE_MAP_ ## type, ptr) +#define PB_ANONYMOUS_ONEOF_STATIC(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_STATIC | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, m), \ + pb_membersize(st, m), 0, ptr} + +#define PB_ANONYMOUS_ONEOF_POINTER(u, tag, st, m, fd, ltype, ptr) \ + {tag, PB_ATYPE_POINTER | PB_HTYPE_ONEOF | ltype, \ + fd, pb_delta(st, which_ ## u, m), \ + pb_membersize(st, m[0]), 0, ptr} + +#define PB_ANONYMOUS_ONEOF_FIELD(union_name, tag, type, rules, allocation, placement, message, field, prevfield, ptr) \ + PB_ANONYMOUS_ONEOF_ ## allocation(union_name, tag, message, field, \ + PB_DATAOFFSET_ ## placement(message, field, prevfield), \ + PB_LTYPE_MAP_ ## type, ptr) /* These macros are used for giving out error messages. * They are mostly a debugging aid; the main error information * is the true/false return value from functions. * Some code space can be saved by disabling the error * messages if not used. + * + * PB_SET_ERROR() sets the error message if none has been set yet. + * msg must be a constant string literal. + * PB_GET_ERROR() always returns a pointer to a string. + * PB_RETURN_ERROR() sets the error and returns false from current + * function. */ #ifdef PB_NO_ERRMSG -#define PB_RETURN_ERROR(stream,msg) \ - do {\ - UNUSED(stream); \ - return false; \ - } while(0) +#define PB_SET_ERROR(stream, msg) PB_UNUSED(stream) #define PB_GET_ERROR(stream) "(errmsg disabled)" #else -#define PB_RETURN_ERROR(stream,msg) \ - do {\ - if ((stream)->errmsg == NULL) \ - (stream)->errmsg = (msg); \ - return false; \ - } while(0) +#define PB_SET_ERROR(stream, msg) (stream->errmsg = (stream)->errmsg ? (stream)->errmsg : (msg)) #define PB_GET_ERROR(stream) ((stream)->errmsg ? (stream)->errmsg : "(none)") #endif +#define PB_RETURN_ERROR(stream, msg) return PB_SET_ERROR(stream, msg), false + #endif diff --git a/pb_common.c b/pb_common.c new file mode 100644 index 0000000..4fb7186 --- /dev/null +++ b/pb_common.c @@ -0,0 +1,97 @@ +/* pb_common.c: Common support functions for pb_encode.c and pb_decode.c. + * + * 2014 Petteri Aimonen <jpa@kapsi.fi> + */ + +#include "pb_common.h" + +bool pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct) +{ + iter->start = fields; + iter->pos = fields; + iter->required_field_index = 0; + iter->dest_struct = dest_struct; + iter->pData = (char*)dest_struct + iter->pos->data_offset; + iter->pSize = (char*)iter->pData + iter->pos->size_offset; + + return (iter->pos->tag != 0); +} + +bool pb_field_iter_next(pb_field_iter_t *iter) +{ + const pb_field_t *prev_field = iter->pos; + + if (prev_field->tag == 0) + { + /* Handle empty message types, where the first field is already the terminator. + * In other cases, the iter->pos never points to the terminator. */ + return false; + } + + iter->pos++; + + if (iter->pos->tag == 0) + { + /* Wrapped back to beginning, reinitialize */ + (void)pb_field_iter_begin(iter, iter->start, iter->dest_struct); + return false; + } + else + { + /* Increment the pointers based on previous field size */ + size_t prev_size = prev_field->data_size; + + if (PB_HTYPE(prev_field->type) == PB_HTYPE_ONEOF && + PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF && + iter->pos->data_offset == PB_SIZE_MAX) + { + /* Don't advance pointers inside unions */ + return true; + } + else if (PB_ATYPE(prev_field->type) == PB_ATYPE_STATIC && + PB_HTYPE(prev_field->type) == PB_HTYPE_REPEATED) + { + /* In static arrays, the data_size tells the size of a single entry and + * array_size is the number of entries */ + prev_size *= prev_field->array_size; + } + else if (PB_ATYPE(prev_field->type) == PB_ATYPE_POINTER) + { + /* Pointer fields always have a constant size in the main structure. + * The data_size only applies to the dynamically allocated area. */ + prev_size = sizeof(void*); + } + + if (PB_HTYPE(prev_field->type) == PB_HTYPE_REQUIRED) + { + /* Count the required fields, in order to check their presence in the + * decoder. */ + iter->required_field_index++; + } + + iter->pData = (char*)iter->pData + prev_size + iter->pos->data_offset; + iter->pSize = (char*)iter->pData + iter->pos->size_offset; + return true; + } +} + +bool pb_field_iter_find(pb_field_iter_t *iter, uint32_t tag) +{ + const pb_field_t *start = iter->pos; + + do { + if (iter->pos->tag == tag && + PB_LTYPE(iter->pos->type) != PB_LTYPE_EXTENSION) + { + /* Found the wanted field */ + return true; + } + + (void)pb_field_iter_next(iter); + } while (iter->pos != start); + + /* Searched all the way back to start, and found nothing. */ + return false; +} + + diff --git a/pb_common.h b/pb_common.h new file mode 100644 index 0000000..60b3d37 --- /dev/null +++ b/pb_common.h @@ -0,0 +1,42 @@ +/* pb_common.h: Common support functions for pb_encode.c and pb_decode.c. + * These functions are rarely needed by applications directly. + */ + +#ifndef PB_COMMON_H_INCLUDED +#define PB_COMMON_H_INCLUDED + +#include "pb.h" + +#ifdef __cplusplus +extern "C" { +#endif + +/* Iterator for pb_field_t list */ +struct pb_field_iter_s { + const pb_field_t *start; /* Start of the pb_field_t array */ + const pb_field_t *pos; /* Current position of the iterator */ + unsigned required_field_index; /* Zero-based index that counts only the required fields */ + void *dest_struct; /* Pointer to start of the structure */ + void *pData; /* Pointer to current field value */ + void *pSize; /* Pointer to count/has field */ +}; +typedef struct pb_field_iter_s pb_field_iter_t; + +/* Initialize the field iterator structure to beginning. + * Returns false if the message type is empty. */ +bool pb_field_iter_begin(pb_field_iter_t *iter, const pb_field_t *fields, void *dest_struct); + +/* Advance the iterator to the next field. + * Returns false when the iterator wraps back to the first field. */ +bool pb_field_iter_next(pb_field_iter_t *iter); + +/* Advance the iterator until it points at a field with the given tag. + * Returns false if no such field exists. */ +bool pb_field_iter_find(pb_field_iter_t *iter, uint32_t tag); + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif + diff --git a/pb_decode.c b/pb_decode.c index 9a48c60..4b80e81 100644 --- a/pb_decode.c +++ b/pb_decode.c @@ -15,38 +15,27 @@ #include "pb.h" #include "pb_decode.h" +#include "pb_common.h" /************************************** * Declarations internal to this file * **************************************/ -/* Iterator for pb_field_t list */ -typedef struct { - const pb_field_t *start; /* Start of the pb_field_t array */ - const pb_field_t *pos; /* Current position of the iterator */ - unsigned field_index; /* Zero-based index of the field. */ - unsigned required_field_index; /* Zero-based index that counts only the required fields */ - void *dest_struct; /* Pointer to the destination structure to decode to */ - void *pData; /* Pointer where to store current field value */ - void *pSize; /* Pointer where to store the size of current array field */ -} pb_field_iterator_t; - typedef bool (*pb_decoder_t)(pb_istream_t *stream, const pb_field_t *field, void *dest) checkreturn; -static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count); -static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); -static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8_t *buf, size_t *size); -static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct); -static bool pb_field_next(pb_field_iterator_t *iter); -static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag); -static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter); -static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter); -static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter); +static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count); +static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size); +static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); +static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); +static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter); +static void iter_from_extension(pb_field_iter_t *iter, pb_extension_t *extension); static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type); -static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter); -static bool checkreturn find_extension_field(pb_field_iterator_t *iter); +static bool checkreturn decode_extension(pb_istream_t *stream, uint32_t tag, pb_wire_type_t wire_type, pb_field_iter_t *iter); +static bool checkreturn find_extension_field(pb_field_iter_t *iter); +static void pb_field_set_to_default(pb_field_iter_t *iter); static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct); static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_decode_varint32_eof(pb_istream_t *stream, uint32_t *dest, bool *eof); static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest); @@ -54,9 +43,24 @@ static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *f static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t *field, void *dest); +static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest); static bool checkreturn pb_skip_varint(pb_istream_t *stream); static bool checkreturn pb_skip_string(pb_istream_t *stream); +#ifdef PB_ENABLE_MALLOC +static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size); +static bool checkreturn pb_release_union_field(pb_istream_t *stream, pb_field_iter_t *iter); +static void pb_release_single_field(const pb_field_iter_t *iter); +#endif + +#ifdef PB_WITHOUT_64BIT +#define pb_int64_t int32_t +#define pb_uint64_t uint32_t +#else +#define pb_int64_t int64_t +#define pb_uint64_t uint64_t +#endif + /* --- Function pointers to field decoders --- * Order in the array must match pb_action_t LTYPE numbering. */ @@ -70,34 +74,36 @@ static const pb_decoder_t PB_DECODERS[PB_LTYPES_COUNT] = { &pb_dec_bytes, &pb_dec_string, &pb_dec_submessage, - NULL /* extensions */ + NULL, /* extensions */ + &pb_dec_fixed_length_bytes }; /******************************* * pb_istream_t implementation * *******************************/ -static bool checkreturn buf_read(pb_istream_t *stream, uint8_t *buf, size_t count) +static bool checkreturn buf_read(pb_istream_t *stream, pb_byte_t *buf, size_t count) { - uint8_t *source = (uint8_t*)stream->state; - stream->state = source + count; + size_t i; + const pb_byte_t *source = (const pb_byte_t*)stream->state; + stream->state = (pb_byte_t*)stream->state + count; if (buf != NULL) { - while (count--) - *buf++ = *source++; + for (i = 0; i < count; i++) + buf[i] = source[i]; } return true; } -bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) +bool checkreturn pb_read(pb_istream_t *stream, pb_byte_t *buf, size_t count) { #ifndef PB_BUFFER_ONLY if (buf == NULL && stream->callback != buf_read) { /* Skip input bytes */ - uint8_t tmp[16]; + pb_byte_t tmp[16]; while (count > 16) { if (!pb_read(stream, tmp, 16)) @@ -127,7 +133,7 @@ bool checkreturn pb_read(pb_istream_t *stream, uint8_t *buf, size_t count) /* Read a single byte from input stream. buf may not be NULL. * This is an optimization for the varint decoding. */ -static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf) +static bool checkreturn pb_readbyte(pb_istream_t *stream, pb_byte_t *buf) { if (stream->bytes_left == 0) PB_RETURN_ERROR(stream, "end-of-stream"); @@ -136,8 +142,8 @@ static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf) if (!stream->callback(stream, buf, 1)) PB_RETURN_ERROR(stream, "io error"); #else - *buf = *(uint8_t*)stream->state; - stream->state = (uint8_t*)stream->state + 1; + *buf = *(const pb_byte_t*)stream->state; + stream->state = (pb_byte_t*)stream->state + 1; #endif stream->bytes_left--; @@ -145,15 +151,23 @@ static bool checkreturn pb_readbyte(pb_istream_t *stream, uint8_t *buf) return true; } -pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) +pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize) { pb_istream_t stream; + /* Cast away the const from buf without a compiler error. We are + * careful to use it only in a const manner in the callbacks. + */ + union { + void *state; + const void *c_state; + } state; #ifdef PB_BUFFER_ONLY stream.callback = NULL; #else stream.callback = &buf_read; #endif - stream.state = buf; + state.c_state = buf; + stream.state = state.state; stream.bytes_left = bufsize; #ifndef PB_NO_ERRMSG stream.errmsg = NULL; @@ -165,13 +179,23 @@ pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize) * Helper functions * ********************/ -static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) +static bool checkreturn pb_decode_varint32_eof(pb_istream_t *stream, uint32_t *dest, bool *eof) { - uint8_t byte; + pb_byte_t byte; uint32_t result; if (!pb_readbyte(stream, &byte)) + { + if (stream->bytes_left == 0) + { + if (eof) + { + *eof = true; + } + } + return false; + } if ((byte & 0x80) == 0) { @@ -181,30 +205,52 @@ static bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) else { /* Multibyte case */ - uint8_t bitpos = 7; + uint_fast8_t bitpos = 7; result = byte & 0x7F; do { - if (bitpos >= 32) - PB_RETURN_ERROR(stream, "varint overflow"); - if (!pb_readbyte(stream, &byte)) return false; - result |= (uint32_t)(byte & 0x7F) << bitpos; - bitpos = (uint8_t)(bitpos + 7); + if (bitpos >= 32) + { + /* Note: The varint could have trailing 0x80 bytes, or 0xFF for negative. */ + uint8_t sign_extension = (bitpos < 63) ? 0xFF : 0x01; + + if ((byte & 0x7F) != 0x00 && ((result >> 31) == 0 || byte != sign_extension)) + { + PB_RETURN_ERROR(stream, "varint overflow"); + } + } + else + { + result |= (uint32_t)(byte & 0x7F) << bitpos; + } + bitpos = (uint_fast8_t)(bitpos + 7); } while (byte & 0x80); + + if (bitpos == 35 && (byte & 0x70) != 0) + { + /* The last byte was at bitpos=28, so only bottom 4 bits fit. */ + PB_RETURN_ERROR(stream, "varint overflow"); + } } *dest = result; return true; } +bool checkreturn pb_decode_varint32(pb_istream_t *stream, uint32_t *dest) +{ + return pb_decode_varint32_eof(stream, dest, NULL); +} + +#ifndef PB_WITHOUT_64BIT bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) { - uint8_t byte; - uint8_t bitpos = 0; + pb_byte_t byte; + uint_fast8_t bitpos = 0; uint64_t result = 0; do @@ -216,16 +262,17 @@ bool checkreturn pb_decode_varint(pb_istream_t *stream, uint64_t *dest) return false; result |= (uint64_t)(byte & 0x7F) << bitpos; - bitpos = (uint8_t)(bitpos + 7); + bitpos = (uint_fast8_t)(bitpos + 7); } while (byte & 0x80); *dest = result; return true; } +#endif bool checkreturn pb_skip_varint(pb_istream_t *stream) { - uint8_t byte; + pb_byte_t byte; do { if (!pb_read(stream, &byte, 1)) @@ -250,11 +297,8 @@ bool checkreturn pb_decode_tag(pb_istream_t *stream, pb_wire_type_t *wire_type, *wire_type = (pb_wire_type_t) 0; *tag = 0; - if (!pb_decode_varint32(stream, &temp)) + if (!pb_decode_varint32_eof(stream, &temp, eof)) { - if (stream->bytes_left == 0) - *eof = true; - return false; } @@ -284,7 +328,7 @@ bool checkreturn pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type) /* Read a raw value to buffer, for the purpose of passing it to callback as * a substream. Size is maximum size on call, and actual size on return. */ -static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, uint8_t *buf, size_t *size) +static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire_type, pb_byte_t *buf, size_t *size) { size_t max_size = *size; switch (wire_type) @@ -307,6 +351,12 @@ static bool checkreturn read_raw_value(pb_istream_t *stream, pb_wire_type_t wire *size = 4; return pb_read(stream, buf, 4); + case PB_WT_STRING: + /* Calling read_raw_value with a PB_WT_STRING is an error. + * Explicitly handle this case and fallthrough to default to avoid + * compiler warnings. + */ + default: PB_RETURN_ERROR(stream, "invalid wire_type"); } } @@ -329,84 +379,26 @@ bool checkreturn pb_make_string_substream(pb_istream_t *stream, pb_istream_t *su return true; } -void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) +bool checkreturn pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream) { + if (substream->bytes_left) { + if (!pb_read(substream, NULL, substream->bytes_left)) + return false; + } + stream->state = substream->state; #ifndef PB_NO_ERRMSG stream->errmsg = substream->errmsg; #endif -} - -static void pb_field_init(pb_field_iterator_t *iter, const pb_field_t *fields, void *dest_struct) -{ - iter->start = iter->pos = fields; - iter->field_index = 0; - iter->required_field_index = 0; - iter->pData = (char*)dest_struct + iter->pos->data_offset; - iter->pSize = (char*)iter->pData + iter->pos->size_offset; - iter->dest_struct = dest_struct; -} - -static bool pb_field_next(pb_field_iterator_t *iter) -{ - bool notwrapped = true; - size_t prev_size = iter->pos->data_size; - - if (PB_ATYPE(iter->pos->type) == PB_ATYPE_STATIC && - PB_HTYPE(iter->pos->type) == PB_HTYPE_REPEATED) - { - prev_size *= iter->pos->array_size; - } - else if (PB_ATYPE(iter->pos->type) == PB_ATYPE_POINTER) - { - prev_size = sizeof(void*); - } - - if (iter->pos->tag == 0) - return false; /* Only happens with empty message types */ - - if (PB_HTYPE(iter->pos->type) == PB_HTYPE_REQUIRED) - iter->required_field_index++; - - iter->pos++; - iter->field_index++; - if (iter->pos->tag == 0) - { - iter->pos = iter->start; - iter->field_index = 0; - iter->required_field_index = 0; - iter->pData = iter->dest_struct; - prev_size = 0; - notwrapped = false; - } - - iter->pData = (char*)iter->pData + prev_size + iter->pos->data_offset; - iter->pSize = (char*)iter->pData + iter->pos->size_offset; - return notwrapped; -} - -static bool checkreturn pb_field_find(pb_field_iterator_t *iter, uint32_t tag) -{ - unsigned start = iter->field_index; - - do { - if (iter->pos->tag == tag && - PB_LTYPE(iter->pos->type) != PB_LTYPE_EXTENSION) - { - return true; - } - (void)pb_field_next(iter); - } while (iter->field_index != start); - - return false; + return true; } /************************* * Decode a single field * *************************/ -static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter) { pb_type_t type; pb_decoder_t func; @@ -420,7 +412,8 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t return func(stream, iter->pos, iter->pData); case PB_HTYPE_OPTIONAL: - *(bool*)iter->pSize = true; + if (iter->pSize != iter->pData) + *(bool*)iter->pSize = true; return func(stream, iter->pos, iter->pData); case PB_HTYPE_REPEATED: @@ -429,14 +422,15 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t { /* Packed array */ bool status = true; - size_t *size = (size_t*)iter->pSize; + pb_size_t *size = (pb_size_t*)iter->pSize; + pb_istream_t substream; if (!pb_make_string_substream(stream, &substream)) return false; - + while (substream.bytes_left > 0 && *size < iter->pos->array_size) { - void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); + void *pItem = (char*)iter->pData + iter->pos->data_size * (*size); if (!func(&substream, iter->pos, pItem)) { status = false; @@ -444,25 +438,37 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t } (*size)++; } - pb_close_string_substream(stream, &substream); - + if (substream.bytes_left != 0) PB_RETURN_ERROR(stream, "array overflow"); - + if (!pb_close_string_substream(stream, &substream)) + return false; + return status; } else { /* Repeated field */ - size_t *size = (size_t*)iter->pSize; - void *pItem = (uint8_t*)iter->pData + iter->pos->data_size * (*size); - if (*size >= iter->pos->array_size) + pb_size_t *size = (pb_size_t*)iter->pSize; + char *pItem = (char*)iter->pData + iter->pos->data_size * (*size); + + if ((*size)++ >= iter->pos->array_size) PB_RETURN_ERROR(stream, "array overflow"); - - (*size)++; + return func(stream, iter->pos, pItem); } + case PB_HTYPE_ONEOF: + *(pb_size_t*)iter->pSize = iter->pos->tag; + if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) + { + /* We memset to zero so that any callbacks are set to NULL. + * Then set any default values. */ + memset(iter->pData, 0, iter->pos->data_size); + pb_message_set_to_defaults((const pb_field_t*)iter->pos->ptr, iter->pData); + } + return func(stream, iter->pos, iter->pData); + default: PB_RETURN_ERROR(stream, "invalid field type"); } @@ -470,16 +476,37 @@ static bool checkreturn decode_static_field(pb_istream_t *stream, pb_wire_type_t #ifdef PB_ENABLE_MALLOC /* Allocate storage for the field and store the pointer at iter->pData. - * array_size is the number of entries to reserve in an array. */ + * array_size is the number of entries to reserve in an array. + * Zero size is not allowed, use pb_free() for releasing. + */ static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t data_size, size_t array_size) { void *ptr = *(void**)pData; - size_t size = array_size * data_size; + + if (data_size == 0 || array_size == 0) + PB_RETURN_ERROR(stream, "invalid size"); + + /* Check for multiplication overflows. + * This code avoids the costly division if the sizes are small enough. + * Multiplication is safe as long as only half of bits are set + * in either multiplicand. + */ + { + const size_t check_limit = (size_t)1 << (sizeof(size_t) * 4); + if (data_size >= check_limit || array_size >= check_limit) + { + const size_t size_max = (size_t)-1; + if (size_max / array_size < data_size) + { + PB_RETURN_ERROR(stream, "size too large"); + } + } + } /* Allocate new or expand previous allocation */ /* Note: on failure the old pointer will remain in the structure, * the message must be freed by caller also on error return. */ - ptr = pb_realloc(ptr, size); + ptr = pb_realloc(ptr, array_size * data_size); if (ptr == NULL) PB_RETURN_ERROR(stream, "realloc failed"); @@ -488,7 +515,7 @@ static bool checkreturn allocate_field(pb_istream_t *stream, void *pData, size_t } /* Clear a newly allocated item in case it contains a pointer, or is a submessage. */ -static void initialize_pointer_field(void *pItem, pb_field_iterator_t *iter) +static void initialize_pointer_field(void *pItem, pb_field_iter_t *iter) { if (PB_LTYPE(iter->pos->type) == PB_LTYPE_STRING || PB_LTYPE(iter->pos->type) == PB_LTYPE_BYTES) @@ -497,16 +524,19 @@ static void initialize_pointer_field(void *pItem, pb_field_iterator_t *iter) } else if (PB_LTYPE(iter->pos->type) == PB_LTYPE_SUBMESSAGE) { + /* We memset to zero so that any callbacks are set to NULL. + * Then set any default values. */ + memset(pItem, 0, iter->pos->data_size); pb_message_set_to_defaults((const pb_field_t *) iter->pos->ptr, pItem); } } #endif -static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter) { #ifndef PB_ENABLE_MALLOC - UNUSED(wire_type); - UNUSED(iter); + PB_UNUSED(wire_type); + PB_UNUSED(iter); PB_RETURN_ERROR(stream, "no malloc support"); #else pb_type_t type; @@ -519,6 +549,19 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ { case PB_HTYPE_REQUIRED: case PB_HTYPE_OPTIONAL: + case PB_HTYPE_ONEOF: + if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE && + *(void**)iter->pData != NULL) + { + /* Duplicate field, have to release the old allocation first. */ + pb_release_single_field(iter); + } + + if (PB_HTYPE(type) == PB_HTYPE_ONEOF) + { + *(pb_size_t*)iter->pSize = iter->pos->tag; + } + if (PB_LTYPE(type) == PB_LTYPE_STRING || PB_LTYPE(type) == PB_LTYPE_BYTES) { @@ -539,7 +582,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ { /* Packed array, multiple items come in at once. */ bool status = true; - size_t *size = (size_t*)iter->pSize; + pb_size_t *size = (pb_size_t*)iter->pSize; size_t allocated_size = *size; void *pItem; pb_istream_t substream; @@ -549,7 +592,7 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ while (substream.bytes_left) { - if (*size + 1 > allocated_size) + if ((size_t)*size + 1 > allocated_size) { /* Allocate more storage. This tries to guess the * number of remaining entries. Round the division @@ -564,41 +607,55 @@ static bool checkreturn decode_pointer_field(pb_istream_t *stream, pb_wire_type_ } /* Decode the array entry */ - pItem = *(uint8_t**)iter->pData + iter->pos->data_size * (*size); + pItem = *(char**)iter->pData + iter->pos->data_size * (*size); initialize_pointer_field(pItem, iter); if (!func(&substream, iter->pos, pItem)) { status = false; break; } + + if (*size == PB_SIZE_MAX) + { +#ifndef PB_NO_ERRMSG + stream->errmsg = "too many array entries"; +#endif + status = false; + break; + } + (*size)++; } - pb_close_string_substream(stream, &substream); + if (!pb_close_string_substream(stream, &substream)) + return false; return status; } else { /* Normal repeated field, i.e. only one item at a time. */ - size_t *size = (size_t*)iter->pSize; + pb_size_t *size = (pb_size_t*)iter->pSize; void *pItem; + if (*size == PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "too many array entries"); + (*size)++; if (!allocate_field(stream, iter->pData, iter->pos->data_size, *size)) return false; - pItem = *(uint8_t**)iter->pData + iter->pos->data_size * (*size - 1); + pItem = *(char**)iter->pData + iter->pos->data_size * (*size - 1); initialize_pointer_field(pItem, iter); return func(stream, iter->pos, pItem); } - + default: PB_RETURN_ERROR(stream, "invalid field type"); } #endif } -static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter) { pb_callback_t *pCallback = (pb_callback_t*)iter->pData; @@ -608,7 +665,7 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type void **arg = &(pCallback->arg); #endif - if (pCallback->funcs.decode == NULL) + if (pCallback == NULL || pCallback->funcs.decode == NULL) return pb_skip_field(stream, wire_type); if (wire_type == PB_WT_STRING) @@ -624,7 +681,9 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type PB_RETURN_ERROR(stream, "callback failed"); } while (substream.bytes_left); - pb_close_string_substream(stream, &substream); + if (!pb_close_string_substream(stream, &substream)) + return false; + return true; } else @@ -634,7 +693,7 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type * which in turn allows to use same callback for packed and * not-packed fields. */ pb_istream_t substream; - uint8_t buffer[10]; + pb_byte_t buffer[10]; size_t size = sizeof(buffer); if (!read_raw_value(stream, wire_type, buffer, &size)) @@ -645,8 +704,18 @@ static bool checkreturn decode_callback_field(pb_istream_t *stream, pb_wire_type } } -static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iterator_t *iter) +static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_type, pb_field_iter_t *iter) { +#ifdef PB_ENABLE_MALLOC + /* When decoding an oneof field, check if there is old data that must be + * released first. */ + if (PB_HTYPE(iter->pos->type) == PB_HTYPE_ONEOF) + { + if (!pb_release_union_field(stream, iter)) + return false; + } +#endif + switch (PB_ATYPE(iter->pos->type)) { case PB_ATYPE_STATIC: @@ -663,32 +732,45 @@ static bool checkreturn decode_field(pb_istream_t *stream, pb_wire_type_t wire_t } } +static void iter_from_extension(pb_field_iter_t *iter, pb_extension_t *extension) +{ + /* Fake a field iterator for the extension field. + * It is not actually safe to advance this iterator, but decode_field + * will not even try to. */ + const pb_field_t *field = (const pb_field_t*)extension->type->arg; + (void)pb_field_iter_begin(iter, field, extension->dest); + iter->pData = extension->dest; + iter->pSize = &extension->found; + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + { + /* For pointer extensions, the pointer is stored directly + * in the extension structure. This avoids having an extra + * indirection. */ + iter->pData = &extension->dest; + } +} + /* Default handler for extension fields. Expects a pb_field_t structure * in extension->type->arg. */ static bool checkreturn default_extension_decoder(pb_istream_t *stream, pb_extension_t *extension, uint32_t tag, pb_wire_type_t wire_type) { const pb_field_t *field = (const pb_field_t*)extension->type->arg; - pb_field_iterator_t iter; + pb_field_iter_t iter; if (field->tag != tag) return true; - iter.start = field; - iter.pos = field; - iter.field_index = 0; - iter.required_field_index = 0; - iter.dest_struct = extension->dest; - iter.pData = extension->dest; - iter.pSize = &extension->found; - + iter_from_extension(&iter, extension); + extension->found = true; return decode_field(stream, wire_type, &iter); } /* Try to decode an unknown field as an extension field. Tries each extension * decoder in turn, until one of them handles the field or loop ends. */ static bool checkreturn decode_extension(pb_istream_t *stream, - uint32_t tag, pb_wire_type_t wire_type, pb_field_iterator_t *iter) + uint32_t tag, pb_wire_type_t wire_type, pb_field_iter_t *iter) { pb_extension_t *extension = *(pb_extension_t* const *)iter->pData; size_t pos = stream->bytes_left; @@ -713,81 +795,103 @@ static bool checkreturn decode_extension(pb_istream_t *stream, /* Step through the iterator until an extension field is found or until all * entries have been checked. There can be only one extension field per * message. Returns false if no extension field is found. */ -static bool checkreturn find_extension_field(pb_field_iterator_t *iter) +static bool checkreturn find_extension_field(pb_field_iter_t *iter) { - unsigned start = iter->field_index; + const pb_field_t *start = iter->pos; do { if (PB_LTYPE(iter->pos->type) == PB_LTYPE_EXTENSION) return true; - (void)pb_field_next(iter); - } while (iter->field_index != start); + (void)pb_field_iter_next(iter); + } while (iter->pos != start); return false; } /* Initialize message fields to default values, recursively */ -static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct) +static void pb_field_set_to_default(pb_field_iter_t *iter) { - pb_field_iterator_t iter; - pb_field_init(&iter, fields, dest_struct); + pb_type_t type; + type = iter->pos->type; - do + if (PB_LTYPE(type) == PB_LTYPE_EXTENSION) { - pb_type_t type; - type = iter.pos->type; - - /* Avoid crash on empty message types (zero fields) */ - if (iter.pos->tag == 0) - continue; - - if (PB_ATYPE(type) == PB_ATYPE_STATIC) + pb_extension_t *ext = *(pb_extension_t* const *)iter->pData; + while (ext != NULL) { - if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL) - { - /* Set has_field to false. Still initialize the optional field - * itself also. */ - *(bool*)iter.pSize = false; - } - else if (PB_HTYPE(type) == PB_HTYPE_REPEATED) - { - /* Set array count to 0, no need to initialize contents. */ - *(size_t*)iter.pSize = 0; - continue; - } - - if (PB_LTYPE(iter.pos->type) == PB_LTYPE_SUBMESSAGE) + pb_field_iter_t ext_iter; + ext->found = false; + iter_from_extension(&ext_iter, ext); + pb_field_set_to_default(&ext_iter); + ext = ext->next; + } + } + else if (PB_ATYPE(type) == PB_ATYPE_STATIC) + { + bool init_data = true; + if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL && iter->pSize != iter->pData) + { + /* Set has_field to false. Still initialize the optional field + * itself also. */ + *(bool*)iter->pSize = false; + } + else if (PB_HTYPE(type) == PB_HTYPE_REPEATED || + PB_HTYPE(type) == PB_HTYPE_ONEOF) + { + /* REPEATED: Set array count to 0, no need to initialize contents. + ONEOF: Set which_field to 0. */ + *(pb_size_t*)iter->pSize = 0; + init_data = false; + } + + if (init_data) + { + if (PB_LTYPE(iter->pos->type) == PB_LTYPE_SUBMESSAGE) { /* Initialize submessage to defaults */ - pb_message_set_to_defaults((const pb_field_t *) iter.pos->ptr, iter.pData); + pb_message_set_to_defaults((const pb_field_t *) iter->pos->ptr, iter->pData); } - else if (iter.pos->ptr != NULL) + else if (iter->pos->ptr != NULL) { /* Initialize to default value */ - memcpy(iter.pData, iter.pos->ptr, iter.pos->data_size); + memcpy(iter->pData, iter->pos->ptr, iter->pos->data_size); } else { /* Initialize to zeros */ - memset(iter.pData, 0, iter.pos->data_size); + memset(iter->pData, 0, iter->pos->data_size); } } - else if (PB_ATYPE(type) == PB_ATYPE_POINTER) - { - /* Initialize the pointer to NULL. */ - *(void**)iter.pData = NULL; - - /* Initialize array count to 0. */ - if (PB_HTYPE(type) == PB_HTYPE_REPEATED) - { - *(size_t*)iter.pSize = 0; - } - } - else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK) + } + else if (PB_ATYPE(type) == PB_ATYPE_POINTER) + { + /* Initialize the pointer to NULL. */ + *(void**)iter->pData = NULL; + + /* Initialize array count to 0. */ + if (PB_HTYPE(type) == PB_HTYPE_REPEATED || + PB_HTYPE(type) == PB_HTYPE_ONEOF) { - /* Don't overwrite callback */ + *(pb_size_t*)iter->pSize = 0; } - } while (pb_field_next(&iter)); + } + else if (PB_ATYPE(type) == PB_ATYPE_CALLBACK) + { + /* Don't overwrite callback */ + } +} + +static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_struct) +{ + pb_field_iter_t iter; + + if (!pb_field_iter_begin(&iter, fields, dest_struct)) + return; /* Empty message type */ + + do + { + pb_field_set_to_default(&iter); + } while (pb_field_iter_next(&iter)); } /********************* @@ -796,18 +900,28 @@ static void pb_message_set_to_defaults(const pb_field_t fields[], void *dest_str bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { - uint8_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 7) / 8] = {0, 0, 0, 0, 0, 0, 0, 0}; + uint32_t fields_seen[(PB_MAX_REQUIRED_FIELDS + 31) / 32] = {0, 0}; + const uint32_t allbits = ~(uint32_t)0; uint32_t extension_range_start = 0; - pb_field_iterator_t iter; - - pb_field_init(&iter, fields, dest_struct); - + pb_field_iter_t iter; + + /* 'fixed_count_field' and 'fixed_count_size' track position of a repeated fixed + * count field. This can only handle _one_ repeated fixed count field that + * is unpacked and unordered among other (non repeated fixed count) fields. + */ + const pb_field_t *fixed_count_field = NULL; + pb_size_t fixed_count_size = 0; + + /* Return value ignored, as empty message types will be correctly handled by + * pb_field_iter_find() anyway. */ + (void)pb_field_iter_begin(&iter, fields, dest_struct); + while (stream->bytes_left) { uint32_t tag; pb_wire_type_t wire_type; bool eof; - + if (!pb_decode_tag(stream, &wire_type, &tag, &eof)) { if (eof) @@ -815,8 +929,8 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ else return false; } - - if (!pb_field_find(&iter, tag)) + + if (!pb_field_iter_find(&iter, tag)) { /* No match found, check if it matches an extension. */ if (tag >= extension_range_start) @@ -825,38 +939,70 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ extension_range_start = (uint32_t)-1; else extension_range_start = iter.pos->tag; - + if (tag >= extension_range_start) { size_t pos = stream->bytes_left; - + if (!decode_extension(stream, tag, wire_type, &iter)) return false; - + if (pos != stream->bytes_left) { /* The field was handled */ - continue; + continue; } } } - + /* No match found, skip data */ if (!pb_skip_field(stream, wire_type)) return false; continue; } - + + /* If a repeated fixed count field was found, get size from + * 'fixed_count_field' as there is no counter contained in the struct. + */ + if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REPEATED + && iter.pSize == iter.pData) + { + if (fixed_count_field != iter.pos) { + /* If the new fixed count field does not match the previous one, + * check that the previous one is NULL or that it finished + * receiving all the expected data. + */ + if (fixed_count_field != NULL && + fixed_count_size != fixed_count_field->array_size) + { + PB_RETURN_ERROR(stream, "wrong size for fixed count field"); + } + + fixed_count_field = iter.pos; + fixed_count_size = 0; + } + + iter.pSize = &fixed_count_size; + } + if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REQUIRED && iter.required_field_index < PB_MAX_REQUIRED_FIELDS) { - fields_seen[iter.required_field_index >> 3] |= (uint8_t)(1 << (iter.required_field_index & 7)); + uint32_t tmp = ((uint32_t)1 << (iter.required_field_index & 31)); + fields_seen[iter.required_field_index >> 5] |= tmp; } - + if (!decode_field(stream, wire_type, &iter)) return false; } - + + /* Check that all elements of the last decoded fixed count field were present. */ + if (fixed_count_field != NULL && + fixed_count_size != fixed_count_field->array_size) + { + PB_RETURN_ERROR(stream, "wrong size for fixed count field"); + } + /* Check that all required fields were present. */ { /* First figure out the number of required fields by @@ -869,22 +1015,34 @@ bool checkreturn pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[ do { req_field_count = iter.required_field_index; last_type = iter.pos->type; - } while (pb_field_next(&iter)); + } while (pb_field_iter_next(&iter)); /* Fixup if last field was also required. */ if (PB_HTYPE(last_type) == PB_HTYPE_REQUIRED && iter.pos->tag != 0) req_field_count++; - /* Check the whole bytes */ - for (i = 0; i < (req_field_count >> 3); i++) + if (req_field_count > PB_MAX_REQUIRED_FIELDS) + req_field_count = PB_MAX_REQUIRED_FIELDS; + + if (req_field_count > 0) { - if (fields_seen[i] != 0xFF) - PB_RETURN_ERROR(stream, "missing required field"); + /* Check the whole words */ + for (i = 0; i < (req_field_count >> 5); i++) + { + if (fields_seen[i] != allbits) + PB_RETURN_ERROR(stream, "missing required field"); + } + + /* Check the remaining bits (if any) */ + if ((req_field_count & 31) != 0) + { + if (fields_seen[req_field_count >> 5] != + (allbits >> (32 - (req_field_count & 31)))) + { + PB_RETURN_ERROR(stream, "missing required field"); + } + } } - - /* Check the remaining bits */ - if (fields_seen[req_field_count >> 3] != (0xFF >> (8 - (req_field_count & 7)))) - PB_RETURN_ERROR(stream, "missing required field"); } return true; @@ -904,6 +1062,21 @@ bool checkreturn pb_decode(pb_istream_t *stream, const pb_field_t fields[], void return status; } +bool pb_decode_delimited_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + pb_istream_t substream; + bool status; + + if (!pb_make_string_substream(stream, &substream)) + return false; + + status = pb_decode_noinit(&substream, fields, dest_struct); + + if (!pb_close_string_substream(stream, &substream)) + return false; + return status; +} + bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) { pb_istream_t substream; @@ -913,212 +1086,342 @@ bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void * return false; status = pb_decode(&substream, fields, dest_struct); - pb_close_string_substream(stream, &substream); + + if (!pb_close_string_substream(stream, &substream)) + return false; return status; } +bool pb_decode_nullterminated(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct) +{ + /* This behaviour will be separated in nanopb-0.4.0, see issue #278. */ + return pb_decode(stream, fields, dest_struct); +} + #ifdef PB_ENABLE_MALLOC -void pb_release(const pb_field_t fields[], void *dest_struct) +/* Given an oneof field, if there has already been a field inside this oneof, + * release it before overwriting with a different one. */ +static bool pb_release_union_field(pb_istream_t *stream, pb_field_iter_t *iter) { - pb_field_iterator_t iter; - pb_field_init(&iter, fields, dest_struct); + pb_size_t old_tag = *(pb_size_t*)iter->pSize; /* Previous which_ value */ + pb_size_t new_tag = iter->pos->tag; /* New which_ value */ + + if (old_tag == 0) + return true; /* Ok, no old data in union */ + + if (old_tag == new_tag) + return true; /* Ok, old data is of same type => merge */ + + /* Release old data. The find can fail if the message struct contains + * invalid data. */ + if (!pb_field_iter_find(iter, old_tag)) + PB_RETURN_ERROR(stream, "invalid union tag"); + + pb_release_single_field(iter); + + /* Restore iterator to where it should be. + * This shouldn't fail unless the pb_field_t structure is corrupted. */ + if (!pb_field_iter_find(iter, new_tag)) + PB_RETURN_ERROR(stream, "iterator error"); - do + return true; +} + +static void pb_release_single_field(const pb_field_iter_t *iter) +{ + pb_type_t type; + type = iter->pos->type; + + if (PB_HTYPE(type) == PB_HTYPE_ONEOF) { - pb_type_t type; - type = iter.pos->type; - - /* Avoid crash on empty message types (zero fields) */ - if (iter.pos->tag == 0) - continue; + if (*(pb_size_t*)iter->pSize != iter->pos->tag) + return; /* This is not the current field in the union */ + } + + /* Release anything contained inside an extension or submsg. + * This has to be done even if the submsg itself is statically + * allocated. */ + if (PB_LTYPE(type) == PB_LTYPE_EXTENSION) + { + /* Release fields from all extensions in the linked list */ + pb_extension_t *ext = *(pb_extension_t**)iter->pData; + while (ext != NULL) + { + pb_field_iter_t ext_iter; + iter_from_extension(&ext_iter, ext); + pb_release_single_field(&ext_iter); + ext = ext->next; + } + } + else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) + { + /* Release fields in submessage or submsg array */ + void *pItem = iter->pData; + pb_size_t count = 1; if (PB_ATYPE(type) == PB_ATYPE_POINTER) { - if (PB_HTYPE(type) == PB_HTYPE_REPEATED && - (PB_LTYPE(type) == PB_LTYPE_STRING || - PB_LTYPE(type) == PB_LTYPE_BYTES)) + pItem = *(void**)iter->pData; + } + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + if (PB_ATYPE(type) == PB_ATYPE_STATIC && iter->pSize == iter->pData) { + /* No _count field so use size of the array */ + count = iter->pos->array_size; + } else { + count = *(pb_size_t*)iter->pSize; + } + + if (PB_ATYPE(type) == PB_ATYPE_STATIC && count > iter->pos->array_size) { - /* Release entries in repeated string or bytes array */ - void **pItem = *(void***)iter.pData; - size_t count = *(size_t*)iter.pSize; - while (count--) - { - pb_free(*pItem); - *pItem++ = NULL; - } + /* Protect against corrupted _count fields */ + count = iter->pos->array_size; } - else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) + } + + if (pItem) + { + while (count--) { - /* Release fields in submessages */ - void *pItem = *(void**)iter.pData; - size_t count = (pItem ? 1 : 0); - - if (PB_HTYPE(type) == PB_HTYPE_REPEATED) - { - count = *(size_t*)iter.pSize; - } - - while (count--) - { - pb_release((const pb_field_t*)iter.pos->ptr, pItem); - pItem = (uint8_t*)pItem + iter.pos->data_size; - } + pb_release((const pb_field_t*)iter->pos->ptr, pItem); + pItem = (char*)pItem + iter->pos->data_size; + } + } + } + + if (PB_ATYPE(type) == PB_ATYPE_POINTER) + { + if (PB_HTYPE(type) == PB_HTYPE_REPEATED && + (PB_LTYPE(type) == PB_LTYPE_STRING || + PB_LTYPE(type) == PB_LTYPE_BYTES)) + { + /* Release entries in repeated string or bytes array */ + void **pItem = *(void***)iter->pData; + pb_size_t count = *(pb_size_t*)iter->pSize; + while (count--) + { + pb_free(*pItem); + *pItem++ = NULL; } - - /* Release main item */ - pb_free(*(void**)iter.pData); - *(void**)iter.pData = NULL; } - } while (pb_field_next(&iter)); + + if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + /* We are going to release the array, so set the size to 0 */ + *(pb_size_t*)iter->pSize = 0; + } + + /* Release main item */ + pb_free(*(void**)iter->pData); + *(void**)iter->pData = NULL; + } +} + +void pb_release(const pb_field_t fields[], void *dest_struct) +{ + pb_field_iter_t iter; + + if (!dest_struct) + return; /* Ignore NULL pointers, similar to free() */ + + if (!pb_field_iter_begin(&iter, fields, dest_struct)) + return; /* Empty message type */ + + do + { + pb_release_single_field(&iter); + } while (pb_field_iter_next(&iter)); } #endif /* Field decoders */ -bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest) +bool pb_decode_svarint(pb_istream_t *stream, pb_int64_t *dest) { - uint64_t value; + pb_uint64_t value; if (!pb_decode_varint(stream, &value)) return false; if (value & 1) - *dest = (int64_t)(~(value >> 1)); + *dest = (pb_int64_t)(~(value >> 1)); else - *dest = (int64_t)(value >> 1); + *dest = (pb_int64_t)(value >> 1); return true; } bool pb_decode_fixed32(pb_istream_t *stream, void *dest) { - #ifdef __BIG_ENDIAN__ - uint8_t *bytes = (uint8_t*)dest; - uint8_t lebytes[4]; - - if (!pb_read(stream, lebytes, 4)) + pb_byte_t bytes[4]; + + if (!pb_read(stream, bytes, 4)) return false; - bytes[0] = lebytes[3]; - bytes[1] = lebytes[2]; - bytes[2] = lebytes[1]; - bytes[3] = lebytes[0]; + *(uint32_t*)dest = ((uint32_t)bytes[0] << 0) | + ((uint32_t)bytes[1] << 8) | + ((uint32_t)bytes[2] << 16) | + ((uint32_t)bytes[3] << 24); return true; - #else - return pb_read(stream, (uint8_t*)dest, 4); - #endif } +#ifndef PB_WITHOUT_64BIT bool pb_decode_fixed64(pb_istream_t *stream, void *dest) { - #ifdef __BIG_ENDIAN__ - uint8_t *bytes = (uint8_t*)dest; - uint8_t lebytes[8]; - - if (!pb_read(stream, lebytes, 8)) + pb_byte_t bytes[8]; + + if (!pb_read(stream, bytes, 8)) return false; - bytes[0] = lebytes[7]; - bytes[1] = lebytes[6]; - bytes[2] = lebytes[5]; - bytes[3] = lebytes[4]; - bytes[4] = lebytes[3]; - bytes[5] = lebytes[2]; - bytes[6] = lebytes[1]; - bytes[7] = lebytes[0]; + *(uint64_t*)dest = ((uint64_t)bytes[0] << 0) | + ((uint64_t)bytes[1] << 8) | + ((uint64_t)bytes[2] << 16) | + ((uint64_t)bytes[3] << 24) | + ((uint64_t)bytes[4] << 32) | + ((uint64_t)bytes[5] << 40) | + ((uint64_t)bytes[6] << 48) | + ((uint64_t)bytes[7] << 56); + return true; - #else - return pb_read(stream, (uint8_t*)dest, 8); - #endif } +#endif static bool checkreturn pb_dec_varint(pb_istream_t *stream, const pb_field_t *field, void *dest) { - uint64_t value; + pb_uint64_t value; + pb_int64_t svalue; + pb_int64_t clamped; if (!pb_decode_varint(stream, &value)) return false; - switch (field->data_size) - { - case 1: *(int8_t*)dest = (int8_t)value; break; - case 2: *(int16_t*)dest = (int16_t)value; break; - case 4: *(int32_t*)dest = (int32_t)value; break; - case 8: *(int64_t*)dest = (int64_t)value; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + /* See issue 97: Google's C++ protobuf allows negative varint values to + * be cast as int32_t, instead of the int64_t that should be used when + * encoding. Previous nanopb versions had a bug in encoding. In order to + * not break decoding of such messages, we cast <=32 bit fields to + * int32_t first to get the sign correct. + */ + if (field->data_size == sizeof(pb_int64_t)) + svalue = (pb_int64_t)value; + else + svalue = (int32_t)value; + + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(pb_int64_t)) + clamped = *(pb_int64_t*)dest = svalue; + else if (field->data_size == sizeof(int32_t)) + clamped = *(int32_t*)dest = (int32_t)svalue; + else if (field->data_size == sizeof(int_least16_t)) + clamped = *(int_least16_t*)dest = (int_least16_t)svalue; + else if (field->data_size == sizeof(int_least8_t)) + clamped = *(int_least8_t*)dest = (int_least8_t)svalue; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + if (clamped != svalue) + PB_RETURN_ERROR(stream, "integer too large"); return true; } static bool checkreturn pb_dec_uvarint(pb_istream_t *stream, const pb_field_t *field, void *dest) { - uint64_t value; + pb_uint64_t value, clamped; if (!pb_decode_varint(stream, &value)) return false; - switch (field->data_size) - { - case 4: *(uint32_t*)dest = (uint32_t)value; break; - case 8: *(uint64_t*)dest = value; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(pb_uint64_t)) + clamped = *(pb_uint64_t*)dest = value; + else if (field->data_size == sizeof(uint32_t)) + clamped = *(uint32_t*)dest = (uint32_t)value; + else if (field->data_size == sizeof(uint_least16_t)) + clamped = *(uint_least16_t*)dest = (uint_least16_t)value; + else if (field->data_size == sizeof(uint_least8_t)) + clamped = *(uint_least8_t*)dest = (uint_least8_t)value; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + if (clamped != value) + PB_RETURN_ERROR(stream, "integer too large"); + return true; } static bool checkreturn pb_dec_svarint(pb_istream_t *stream, const pb_field_t *field, void *dest) { - int64_t value; + pb_int64_t value, clamped; if (!pb_decode_svarint(stream, &value)) return false; - switch (field->data_size) - { - case 4: *(int32_t*)dest = (int32_t)value; break; - case 8: *(int64_t*)dest = value; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + /* Cast to the proper field size, while checking for overflows */ + if (field->data_size == sizeof(pb_int64_t)) + clamped = *(pb_int64_t*)dest = value; + else if (field->data_size == sizeof(int32_t)) + clamped = *(int32_t*)dest = (int32_t)value; + else if (field->data_size == sizeof(int_least16_t)) + clamped = *(int_least16_t*)dest = (int_least16_t)value; + else if (field->data_size == sizeof(int_least8_t)) + clamped = *(int_least8_t*)dest = (int_least8_t)value; + else + PB_RETURN_ERROR(stream, "invalid data_size"); + + if (clamped != value) + PB_RETURN_ERROR(stream, "integer too large"); return true; } static bool checkreturn pb_dec_fixed32(pb_istream_t *stream, const pb_field_t *field, void *dest) { - UNUSED(field); + PB_UNUSED(field); return pb_decode_fixed32(stream, dest); } static bool checkreturn pb_dec_fixed64(pb_istream_t *stream, const pb_field_t *field, void *dest) { - UNUSED(field); + PB_UNUSED(field); +#ifndef PB_WITHOUT_64BIT return pb_decode_fixed64(stream, dest); +#else + PB_UNUSED(dest); + PB_RETURN_ERROR(stream, "no 64bit support"); +#endif } static bool checkreturn pb_dec_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) { uint32_t size; + size_t alloc_size; pb_bytes_array_t *bdest; if (!pb_decode_varint32(stream, &size)) return false; + if (size > PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "bytes overflow"); + + alloc_size = PB_BYTES_ARRAY_T_ALLOCSIZE(size); + if (size > alloc_size) + PB_RETURN_ERROR(stream, "size too large"); + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { #ifndef PB_ENABLE_MALLOC PB_RETURN_ERROR(stream, "no malloc support"); #else - if (!allocate_field(stream, dest, PB_BYTES_ARRAY_T_ALLOCSIZE(size), 1)) + if (!allocate_field(stream, dest, alloc_size, 1)) return false; bdest = *(pb_bytes_array_t**)dest; #endif } else { - if (PB_BYTES_ARRAY_T_ALLOCSIZE(size) > field->data_size) + if (alloc_size > field->data_size) PB_RETURN_ERROR(stream, "bytes overflow"); bdest = (pb_bytes_array_t*)dest; } - - bdest->size = size; + + bdest->size = (pb_size_t)size; return pb_read(stream, bdest->bytes, size); } @@ -1133,6 +1436,9 @@ static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *fi /* Space for null terminator */ alloc_size = size + 1; + if (alloc_size < size) + PB_RETURN_ERROR(stream, "size too large"); + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { #ifndef PB_ENABLE_MALLOC @@ -1149,8 +1455,8 @@ static bool checkreturn pb_dec_string(pb_istream_t *stream, const pb_field_t *fi PB_RETURN_ERROR(stream, "string overflow"); } - status = pb_read(stream, (uint8_t*)dest, size); - *((uint8_t*)dest + size) = 0; + status = pb_read(stream, (pb_byte_t*)dest, size); + *((pb_byte_t*)dest + size) = 0; return status; } @@ -1173,6 +1479,30 @@ static bool checkreturn pb_dec_submessage(pb_istream_t *stream, const pb_field_t else status = pb_decode_noinit(&substream, submsg_fields, dest); - pb_close_string_substream(stream, &substream); + if (!pb_close_string_substream(stream, &substream)) + return false; return status; } + +static bool checkreturn pb_dec_fixed_length_bytes(pb_istream_t *stream, const pb_field_t *field, void *dest) +{ + uint32_t size; + + if (!pb_decode_varint32(stream, &size)) + return false; + + if (size > PB_SIZE_MAX) + PB_RETURN_ERROR(stream, "bytes overflow"); + + if (size == 0) + { + /* As a special case, treat empty bytes string as all zeros for fixed_length_bytes. */ + memset(dest, 0, field->data_size); + return true; + } + + if (size != field->data_size) + PB_RETURN_ERROR(stream, "incorrect fixed length bytes size"); + + return pb_read(stream, (pb_byte_t*)dest, field->data_size); +} diff --git a/pb_decode.h b/pb_decode.h index 8dc6740..398b24a 100644 --- a/pb_decode.h +++ b/pb_decode.h @@ -3,8 +3,8 @@ * field descriptions created by nanopb_generator.py. */ -#ifndef _PB_DECODE_H_ -#define _PB_DECODE_H_ +#ifndef PB_DECODE_H_INCLUDED +#define PB_DECODE_H_INCLUDED #include "pb.h" @@ -25,7 +25,7 @@ extern "C" { * is different than from the main stream. Don't use bytes_left to compute * any pointers. */ -struct _pb_istream_t +struct pb_istream_s { #ifdef PB_BUFFER_ONLY /* Callback pointer is not used in buffer-only configuration. @@ -34,7 +34,7 @@ struct _pb_istream_t */ int *callback; #else - bool (*callback)(pb_istream_t *stream, uint8_t *buf, size_t count); + bool (*callback)(pb_istream_t *stream, pb_byte_t *buf, size_t count); #endif void *state; /* Free field for use by callback implementation */ @@ -85,6 +85,18 @@ bool pb_decode_noinit(pb_istream_t *stream, const pb_field_t fields[], void *des */ bool pb_decode_delimited(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); +/* Same as pb_decode_delimited, except that it does not initialize the destination structure. + * See pb_decode_noinit + */ +bool pb_decode_delimited_noinit(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + +/* Same as pb_decode, except allows the message to be terminated with a null byte. + * NOTE: Until nanopb-0.4.0, pb_decode() also allows null-termination. This behaviour + * is not supported in most other protobuf implementations, so pb_decode_delimited() + * is a better option for compatibility. + */ +bool pb_decode_nullterminated(pb_istream_t *stream, const pb_field_t fields[], void *dest_struct); + #ifdef PB_ENABLE_MALLOC /* Release any allocated pointer fields. If you use dynamic allocation, you should * call this for any successfully decoded message when you are done with it. If @@ -103,12 +115,12 @@ void pb_release(const pb_field_t fields[], void *dest_struct); * Alternatively, you can use a custom stream that reads directly from e.g. * a file or a network socket. */ -pb_istream_t pb_istream_from_buffer(uint8_t *buf, size_t bufsize); +pb_istream_t pb_istream_from_buffer(const pb_byte_t *buf, size_t bufsize); /* Function to read from a pb_istream_t. You can use this if you need to * read some custom header data, or to read data in field callbacks. */ -bool pb_read(pb_istream_t *stream, uint8_t *buf, size_t count); +bool pb_read(pb_istream_t *stream, pb_byte_t *buf, size_t count); /************************************************ @@ -124,23 +136,37 @@ bool pb_skip_field(pb_istream_t *stream, pb_wire_type_t wire_type); /* Decode an integer in the varint format. This works for bool, enum, int32, * int64, uint32 and uint64 field types. */ +#ifndef PB_WITHOUT_64BIT bool pb_decode_varint(pb_istream_t *stream, uint64_t *dest); +#else +#define pb_decode_varint pb_decode_varint32 +#endif + +/* Decode an integer in the varint format. This works for bool, enum, int32, + * and uint32 field types. */ +bool pb_decode_varint32(pb_istream_t *stream, uint32_t *dest); /* Decode an integer in the zig-zagged svarint format. This works for sint32 * and sint64. */ +#ifndef PB_WITHOUT_64BIT bool pb_decode_svarint(pb_istream_t *stream, int64_t *dest); +#else +bool pb_decode_svarint(pb_istream_t *stream, int32_t *dest); +#endif /* Decode a fixed32, sfixed32 or float value. You need to pass a pointer to * a 4-byte wide C variable. */ bool pb_decode_fixed32(pb_istream_t *stream, void *dest); +#ifndef PB_WITHOUT_64BIT /* Decode a fixed64, sfixed64 or double value. You need to pass a pointer to * a 8-byte wide C variable. */ bool pb_decode_fixed64(pb_istream_t *stream, void *dest); +#endif /* Make a limited-length substream for reading a PB_WT_STRING field. */ bool pb_make_string_substream(pb_istream_t *stream, pb_istream_t *substream); -void pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); +bool pb_close_string_substream(pb_istream_t *stream, pb_istream_t *substream); #ifdef __cplusplus } /* extern "C" */ diff --git a/pb_encode.c b/pb_encode.c index dc5a273..089172c 100644 --- a/pb_encode.c +++ b/pb_encode.c @@ -5,6 +5,7 @@ #include "pb.h" #include "pb_encode.h" +#include "pb_common.h" /* Use the GCC warn_unused_result attribute to check that all return values * are propagated correctly. On other compilers and gcc before 3.4.0 just @@ -21,11 +22,12 @@ **************************************/ typedef bool (*pb_encoder_t)(pb_ostream_t *stream, const pb_field_t *field, const void *src) checkreturn; -static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); +static bool checkreturn buf_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *field, const void *pData, size_t count, pb_encoder_t func); static bool checkreturn encode_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension); static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData); +static void *pb_const_cast(const void *p); static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src); @@ -34,6 +36,17 @@ static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *f static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src); static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src); +static bool checkreturn pb_enc_fixed_length_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src); + +#ifdef PB_WITHOUT_64BIT +#define pb_int64_t int32_t +#define pb_uint64_t uint32_t + +static bool checkreturn pb_encode_negative_varint(pb_ostream_t *stream, pb_uint64_t value); +#else +#define pb_int64_t int64_t +#define pb_uint64_t uint64_t +#endif /* --- Function pointers to field encoders --- * Order in the array must match pb_action_t LTYPE numbering. @@ -48,25 +61,27 @@ static const pb_encoder_t PB_ENCODERS[PB_LTYPES_COUNT] = { &pb_enc_bytes, &pb_enc_string, &pb_enc_submessage, - NULL /* extensions */ + NULL, /* extensions */ + &pb_enc_fixed_length_bytes }; /******************************* * pb_ostream_t implementation * *******************************/ -static bool checkreturn buf_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) +static bool checkreturn buf_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count) { - uint8_t *dest = (uint8_t*)stream->state; + size_t i; + pb_byte_t *dest = (pb_byte_t*)stream->state; stream->state = dest + count; - while (count--) - *dest++ = *buf++; + for (i = 0; i < count; i++) + dest[i] = buf[i]; return true; } -pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize) +pb_ostream_t pb_ostream_from_buffer(pb_byte_t *buf, size_t bufsize) { pb_ostream_t stream; #ifdef PB_BUFFER_ONLY @@ -83,7 +98,7 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize) return stream; } -bool checkreturn pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count) +bool checkreturn pb_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count) { if (stream->callback != NULL) { @@ -149,7 +164,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie size = sizestream.bytes_written; } - if (!pb_encode_varint(stream, (uint64_t)size)) + if (!pb_encode_varint(stream, (pb_uint64_t)size)) return false; if (stream->callback == NULL) @@ -181,7 +196,7 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie PB_LTYPE(field->type) == PB_LTYPE_BYTES)) { if (!func(stream, field, *(const void* const*)p)) - return false; + return false; } else { @@ -195,21 +210,124 @@ static bool checkreturn encode_array(pb_ostream_t *stream, const pb_field_t *fie return true; } +/* In proto3, all fields are optional and are only encoded if their value is "non-zero". + * This function implements the check for the zero value. */ +static bool pb_check_proto3_default_value(const pb_field_t *field, const void *pData) +{ + pb_type_t type = field->type; + const void *pSize = (const char*)pData + field->size_offset; + + if (PB_HTYPE(type) == PB_HTYPE_REQUIRED) + { + /* Required proto2 fields inside proto3 submessage, pretty rare case */ + return false; + } + else if (PB_HTYPE(type) == PB_HTYPE_REPEATED) + { + /* Repeated fields inside proto3 submessage: present if count != 0 */ + return *(const pb_size_t*)pSize == 0; + } + else if (PB_HTYPE(type) == PB_HTYPE_ONEOF) + { + /* Oneof fields */ + return *(const pb_size_t*)pSize == 0; + } + else if (PB_HTYPE(type) == PB_HTYPE_OPTIONAL && field->size_offset) + { + /* Proto2 optional fields inside proto3 submessage */ + return *(const bool*)pSize == false; + } + + /* Rest is proto3 singular fields */ + + if (PB_ATYPE(type) == PB_ATYPE_STATIC) + { + if (PB_LTYPE(type) == PB_LTYPE_BYTES) + { + const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)pData; + return bytes->size == 0; + } + else if (PB_LTYPE(type) == PB_LTYPE_STRING) + { + return *(const char*)pData == '\0'; + } + else if (PB_LTYPE(type) == PB_LTYPE_FIXED_LENGTH_BYTES) + { + /* Fixed length bytes is only empty if its length is fixed + * as 0. Which would be pretty strange, but we can check + * it anyway. */ + return field->data_size == 0; + } + else if (PB_LTYPE(type) == PB_LTYPE_SUBMESSAGE) + { + /* Check all fields in the submessage to find if any of them + * are non-zero. The comparison cannot be done byte-per-byte + * because the C struct may contain padding bytes that must + * be skipped. + */ + pb_field_iter_t iter; + if (pb_field_iter_begin(&iter, (const pb_field_t*)field->ptr, pb_const_cast(pData))) + { + do + { + if (!pb_check_proto3_default_value(iter.pos, iter.pData)) + { + return false; + } + } while (pb_field_iter_next(&iter)); + } + return true; + } + } + + { + /* Catch-all branch that does byte-per-byte comparison for zero value. + * + * This is for all pointer fields, and for static PB_LTYPE_VARINT, + * UVARINT, SVARINT, FIXED32, FIXED64, EXTENSION fields, and also + * callback fields. These all have integer or pointer value which + * can be compared with 0. + */ + pb_size_t i; + const char *p = (const char*)pData; + for (i = 0; i < field->data_size; i++) + { + if (p[i] != 0) + { + return false; + } + } + + return true; + } +} + /* Encode a field with static or pointer allocation, i.e. one whose data * is available to the encoder directly. */ static bool checkreturn encode_basic_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData) { pb_encoder_t func; - const void *pSize; - bool implicit_has = true; + bool implicit_has; + const void *pSize = &implicit_has; func = PB_ENCODERS[PB_LTYPE(field->type)]; if (field->size_offset) + { + /* Static optional, repeated or oneof field */ pSize = (const char*)pData + field->size_offset; + } + else if (PB_HTYPE(field->type) == PB_HTYPE_OPTIONAL) + { + /* Proto3 style field, optional but without explicit has_ field. */ + implicit_has = !pb_check_proto3_default_value(field, pData); + } else - pSize = &implicit_has; + { + /* Required field, always present */ + implicit_has = true; + } if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) { @@ -217,7 +335,6 @@ static bool checkreturn encode_basic_field(pb_ostream_t *stream, * the data. If the 2nd pointer is NULL, it is interpreted as if * the has_field was false. */ - pData = *(const void* const*)pData; implicit_has = (pData != NULL); } @@ -244,11 +361,29 @@ static bool checkreturn encode_basic_field(pb_ostream_t *stream, } break; - case PB_HTYPE_REPEATED: - if (!encode_array(stream, field, pData, *(const size_t*)pSize, func)) + case PB_HTYPE_REPEATED: { + pb_size_t count; + if (field->size_offset != 0) { + count = *(const pb_size_t*)pSize; + } else { + count = field->array_size; + } + if (!encode_array(stream, field, pData, count, func)) return false; break; + } + case PB_HTYPE_ONEOF: + if (*(const pb_size_t*)pSize == field->tag) + { + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!func(stream, field, pData)) + return false; + } + break; + default: PB_RETURN_ERROR(stream, "invalid field type"); } @@ -301,7 +436,18 @@ static bool checkreturn default_extension_encoder(pb_ostream_t *stream, const pb_extension_t *extension) { const pb_field_t *field = (const pb_field_t*)extension->type->arg; - return encode_field(stream, field, extension->dest); + + if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) + { + /* For pointer extensions, the pointer is stored directly + * in the extension structure. This avoids having an extra + * indirection. */ + return encode_field(stream, field, &extension->dest); + } + else + { + return encode_field(stream, field, extension->dest); + } } /* Walk through all the registered extensions and give them a chance @@ -310,7 +456,7 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream, const pb_field_t *field, const void *pData) { const pb_extension_t *extension = *(const pb_extension_t* const *)pData; - UNUSED(field); + PB_UNUSED(field); while (extension) { @@ -333,42 +479,38 @@ static bool checkreturn encode_extension_field(pb_ostream_t *stream, * Encode all fields * *********************/ +static void *pb_const_cast(const void *p) +{ + /* Note: this casts away const, in order to use the common field iterator + * logic for both encoding and decoding. */ + union { + void *p1; + const void *p2; + } t; + t.p2 = p; + return t.p1; +} + bool checkreturn pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) { - const pb_field_t *field = fields; - const void *pData = src_struct; - size_t prev_size = 0; + pb_field_iter_t iter; + if (!pb_field_iter_begin(&iter, fields, pb_const_cast(src_struct))) + return true; /* Empty message type */ - while (field->tag != 0) - { - pData = (const char*)pData + prev_size + field->data_offset; - if (PB_ATYPE(field->type) == PB_ATYPE_POINTER) - prev_size = sizeof(const void*); - else - prev_size = field->data_size; - - /* Special case for static arrays */ - if (PB_ATYPE(field->type) == PB_ATYPE_STATIC && - PB_HTYPE(field->type) == PB_HTYPE_REPEATED) - { - prev_size *= field->array_size; - } - - if (PB_LTYPE(field->type) == PB_LTYPE_EXTENSION) + do { + if (PB_LTYPE(iter.pos->type) == PB_LTYPE_EXTENSION) { /* Special case for the extension field placeholder */ - if (!encode_extension_field(stream, field, pData)) + if (!encode_extension_field(stream, iter.pos, iter.pData)) return false; } else { /* Regular field */ - if (!encode_field(stream, field, pData)) + if (!encode_field(stream, iter.pos, iter.pData)) return false; } - - field++; - } + } while (pb_field_iter_next(&iter)); return true; } @@ -378,6 +520,16 @@ bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const return pb_encode_submessage(stream, fields, src_struct); } +bool pb_encode_nullterminated(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct) +{ + const pb_byte_t zero = 0; + + if (!pb_encode(stream, fields, src_struct)) + return false; + + return pb_write(stream, &zero, 1); +} + bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *src_struct) { pb_ostream_t stream = PB_OSTREAM_SIZING; @@ -392,17 +544,47 @@ bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *sr /******************** * Helper functions * ********************/ -bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value) + +#ifdef PB_WITHOUT_64BIT +bool checkreturn pb_encode_negative_varint(pb_ostream_t *stream, pb_uint64_t value) +{ + pb_byte_t buffer[10]; + size_t i = 0; + size_t compensation = 32;/* we need to compensate 32 bits all set to 1 */ + + while (value) + { + buffer[i] = (pb_byte_t)((value & 0x7F) | 0x80); + value >>= 7; + if (compensation) + { + /* re-set all the compensation bits we can or need */ + size_t bits = compensation > 7 ? 7 : compensation; + value ^= (pb_uint64_t)((0xFFu >> (8 - bits)) << 25); /* set the number of bits needed on the lowest of the most significant 7 bits */ + compensation -= bits; + } + i++; + } + buffer[i - 1] &= 0x7F; /* Unset top bit on last byte */ + + return pb_write(stream, buffer, i); +} +#endif + +bool checkreturn pb_encode_varint(pb_ostream_t *stream, pb_uint64_t value) { - uint8_t buffer[10]; + pb_byte_t buffer[10]; size_t i = 0; - if (value == 0) - return pb_write(stream, (uint8_t*)&value, 1); + if (value <= 0x7F) + { + pb_byte_t v = (pb_byte_t)value; + return pb_write(stream, &v, 1); + } while (value) { - buffer[i] = (uint8_t)((value & 0x7F) | 0x80); + buffer[i] = (pb_byte_t)((value & 0x7F) | 0x80); value >>= 7; i++; } @@ -411,54 +593,48 @@ bool checkreturn pb_encode_varint(pb_ostream_t *stream, uint64_t value) return pb_write(stream, buffer, i); } -bool checkreturn pb_encode_svarint(pb_ostream_t *stream, int64_t value) +bool checkreturn pb_encode_svarint(pb_ostream_t *stream, pb_int64_t value) { - uint64_t zigzagged; + pb_uint64_t zigzagged; if (value < 0) - zigzagged = ~((uint64_t)value << 1); + zigzagged = ~((pb_uint64_t)value << 1); else - zigzagged = (uint64_t)value << 1; + zigzagged = (pb_uint64_t)value << 1; return pb_encode_varint(stream, zigzagged); } bool checkreturn pb_encode_fixed32(pb_ostream_t *stream, const void *value) { - #ifdef __BIG_ENDIAN__ - const uint8_t *bytes = value; - uint8_t lebytes[4]; - lebytes[0] = bytes[3]; - lebytes[1] = bytes[2]; - lebytes[2] = bytes[1]; - lebytes[3] = bytes[0]; - return pb_write(stream, lebytes, 4); - #else - return pb_write(stream, (const uint8_t*)value, 4); - #endif + uint32_t val = *(const uint32_t*)value; + pb_byte_t bytes[4]; + bytes[0] = (pb_byte_t)(val & 0xFF); + bytes[1] = (pb_byte_t)((val >> 8) & 0xFF); + bytes[2] = (pb_byte_t)((val >> 16) & 0xFF); + bytes[3] = (pb_byte_t)((val >> 24) & 0xFF); + return pb_write(stream, bytes, 4); } +#ifndef PB_WITHOUT_64BIT bool checkreturn pb_encode_fixed64(pb_ostream_t *stream, const void *value) { - #ifdef __BIG_ENDIAN__ - const uint8_t *bytes = value; - uint8_t lebytes[8]; - lebytes[0] = bytes[7]; - lebytes[1] = bytes[6]; - lebytes[2] = bytes[5]; - lebytes[3] = bytes[4]; - lebytes[4] = bytes[3]; - lebytes[5] = bytes[2]; - lebytes[6] = bytes[1]; - lebytes[7] = bytes[0]; - return pb_write(stream, lebytes, 8); - #else - return pb_write(stream, (const uint8_t*)value, 8); - #endif + uint64_t val = *(const uint64_t*)value; + pb_byte_t bytes[8]; + bytes[0] = (pb_byte_t)(val & 0xFF); + bytes[1] = (pb_byte_t)((val >> 8) & 0xFF); + bytes[2] = (pb_byte_t)((val >> 16) & 0xFF); + bytes[3] = (pb_byte_t)((val >> 24) & 0xFF); + bytes[4] = (pb_byte_t)((val >> 32) & 0xFF); + bytes[5] = (pb_byte_t)((val >> 40) & 0xFF); + bytes[6] = (pb_byte_t)((val >> 48) & 0xFF); + bytes[7] = (pb_byte_t)((val >> 56) & 0xFF); + return pb_write(stream, bytes, 8); } +#endif bool checkreturn pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field_number) { - uint64_t tag = ((uint64_t)field_number << 3) | wiretype; + pb_uint64_t tag = ((pb_uint64_t)field_number << 3) | wiretype; return pb_encode_varint(stream, tag); } @@ -484,6 +660,7 @@ bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t case PB_LTYPE_BYTES: case PB_LTYPE_STRING: case PB_LTYPE_SUBMESSAGE: + case PB_LTYPE_FIXED_LENGTH_BYTES: wiretype = PB_WT_STRING; break; @@ -494,9 +671,9 @@ bool checkreturn pb_encode_tag_for_field(pb_ostream_t *stream, const pb_field_t return pb_encode_tag(stream, wiretype, field->tag); } -bool checkreturn pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size) +bool checkreturn pb_encode_string(pb_ostream_t *stream, const pb_byte_t *buffer, size_t size) { - if (!pb_encode_varint(stream, (uint64_t)size)) + if (!pb_encode_varint(stream, (pb_uint64_t)size)) return false; return pb_write(stream, buffer, size); @@ -519,7 +696,7 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie size = substream.bytes_written; - if (!pb_encode_varint(stream, (uint64_t)size)) + if (!pb_encode_varint(stream, (pb_uint64_t)size)) return false; if (stream->callback == NULL) @@ -556,69 +733,89 @@ bool checkreturn pb_encode_submessage(pb_ostream_t *stream, const pb_field_t fie static bool checkreturn pb_enc_varint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - int64_t value = 0; - - /* Cases 1 and 2 are for compilers that have smaller types for bool - * or enums. */ - switch (field->data_size) - { - case 1: value = *(const int8_t*)src; break; - case 2: value = *(const int16_t*)src; break; - case 4: value = *(const int32_t*)src; break; - case 8: value = *(const int64_t*)src; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + pb_int64_t value = 0; + + if (field->data_size == sizeof(int_least8_t)) + value = *(const int_least8_t*)src; + else if (field->data_size == sizeof(int_least16_t)) + value = *(const int_least16_t*)src; + else if (field->data_size == sizeof(int32_t)) + value = *(const int32_t*)src; + else if (field->data_size == sizeof(pb_int64_t)) + value = *(const pb_int64_t*)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); - return pb_encode_varint(stream, (uint64_t)value); +#ifdef PB_WITHOUT_64BIT + if (value < 0) + return pb_encode_negative_varint(stream, (pb_uint64_t)value); + else +#endif + return pb_encode_varint(stream, (pb_uint64_t)value); } static bool checkreturn pb_enc_uvarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - uint64_t value = 0; - - switch (field->data_size) - { - case 4: value = *(const uint32_t*)src; break; - case 8: value = *(const uint64_t*)src; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + pb_uint64_t value = 0; + + if (field->data_size == sizeof(uint_least8_t)) + value = *(const uint_least8_t*)src; + else if (field->data_size == sizeof(uint_least16_t)) + value = *(const uint_least16_t*)src; + else if (field->data_size == sizeof(uint32_t)) + value = *(const uint32_t*)src; + else if (field->data_size == sizeof(pb_uint64_t)) + value = *(const pb_uint64_t*)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); return pb_encode_varint(stream, value); } static bool checkreturn pb_enc_svarint(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - int64_t value = 0; - - switch (field->data_size) - { - case 4: value = *(const int32_t*)src; break; - case 8: value = *(const int64_t*)src; break; - default: PB_RETURN_ERROR(stream, "invalid data_size"); - } + pb_int64_t value = 0; + + if (field->data_size == sizeof(int_least8_t)) + value = *(const int_least8_t*)src; + else if (field->data_size == sizeof(int_least16_t)) + value = *(const int_least16_t*)src; + else if (field->data_size == sizeof(int32_t)) + value = *(const int32_t*)src; + else if (field->data_size == sizeof(pb_int64_t)) + value = *(const pb_int64_t*)src; + else + PB_RETURN_ERROR(stream, "invalid data_size"); return pb_encode_svarint(stream, value); } static bool checkreturn pb_enc_fixed64(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - UNUSED(field); + PB_UNUSED(field); +#ifndef PB_WITHOUT_64BIT return pb_encode_fixed64(stream, src); +#else + PB_UNUSED(src); + PB_RETURN_ERROR(stream, "no 64bit support"); +#endif } static bool checkreturn pb_enc_fixed32(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - UNUSED(field); + PB_UNUSED(field); return pb_encode_fixed32(stream, src); } static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - const pb_bytes_array_t *bytes = (const pb_bytes_array_t*)src; + const pb_bytes_array_t *bytes = NULL; + + bytes = (const pb_bytes_array_t*)src; if (src == NULL) { - /* Threat null pointer as an empty bytes field */ + /* Treat null pointer as an empty bytes field */ return pb_encode_string(stream, NULL, 0); } @@ -633,7 +830,6 @@ static bool checkreturn pb_enc_bytes(pb_ostream_t *stream, const pb_field_t *fie static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *field, const void *src) { - /* strnlen() is not always available, so just use a loop */ size_t size = 0; size_t max_size = field->data_size; const char *p = (const char*)src; @@ -643,10 +839,11 @@ static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *fi if (src == NULL) { - size = 0; /* Threat null pointer as an empty string */ + size = 0; /* Treat null pointer as an empty string */ } else { + /* strnlen() is not always available, so just use a loop */ while (size < max_size && *p != '\0') { size++; @@ -654,7 +851,7 @@ static bool checkreturn pb_enc_string(pb_ostream_t *stream, const pb_field_t *fi } } - return pb_encode_string(stream, (const uint8_t*)src, size); + return pb_encode_string(stream, (const pb_byte_t*)src, size); } static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t *field, const void *src) @@ -665,3 +862,8 @@ static bool checkreturn pb_enc_submessage(pb_ostream_t *stream, const pb_field_t return pb_encode_submessage(stream, (const pb_field_t*)field->ptr, src); } +static bool checkreturn pb_enc_fixed_length_bytes(pb_ostream_t *stream, const pb_field_t *field, const void *src) +{ + return pb_encode_string(stream, (const pb_byte_t*)src, field->data_size); +} + diff --git a/pb_encode.h b/pb_encode.h index f82bac8..8bf78dd 100644 --- a/pb_encode.h +++ b/pb_encode.h @@ -3,8 +3,8 @@ * field descriptions created by nanopb_generator.py. */ -#ifndef _PB_ENCODE_H_ -#define _PB_ENCODE_H_ +#ifndef PB_ENCODE_H_INCLUDED +#define PB_ENCODE_H_INCLUDED #include "pb.h" @@ -24,7 +24,7 @@ extern "C" { * 4) Substreams will modify max_size and bytes_written. Don't use them * to calculate any pointers. */ -struct _pb_ostream_t +struct pb_ostream_s { #ifdef PB_BUFFER_ONLY /* Callback pointer is not used in buffer-only configuration. @@ -35,7 +35,7 @@ struct _pb_ostream_t */ int *callback; #else - bool (*callback)(pb_ostream_t *stream, const uint8_t *buf, size_t count); + bool (*callback)(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); #endif void *state; /* Free field for use by callback implementation. */ size_t max_size; /* Limit number of output bytes written (or use SIZE_MAX). */ @@ -71,6 +71,12 @@ bool pb_encode(pb_ostream_t *stream, const pb_field_t fields[], const void *src_ */ bool pb_encode_delimited(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); +/* Same as pb_encode, but appends a null byte to the message for termination. + * NOTE: This behaviour is not supported in most other protobuf implementations, so pb_encode_delimited() + * is a better option for compatibility. + */ +bool pb_encode_nullterminated(pb_ostream_t *stream, const pb_field_t fields[], const void *src_struct); + /* Encode the message to get the size of the encoded data, but do not store * the data. */ bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *src_struct); @@ -86,7 +92,7 @@ bool pb_get_encoded_size(size_t *size, const pb_field_t fields[], const void *sr * Alternatively, you can use a custom stream that writes directly to e.g. * a file or a network socket. */ -pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); +pb_ostream_t pb_ostream_from_buffer(pb_byte_t *buf, size_t bufsize); /* Pseudo-stream for measuring the size of a message without actually storing * the encoded data. @@ -106,7 +112,7 @@ pb_ostream_t pb_ostream_from_buffer(uint8_t *buf, size_t bufsize); /* Function to write into a pb_ostream_t stream. You can use this if you need * to append or prepend some custom headers to the message. */ -bool pb_write(pb_ostream_t *stream, const uint8_t *buf, size_t count); +bool pb_write(pb_ostream_t *stream, const pb_byte_t *buf, size_t count); /************************************************ @@ -123,22 +129,32 @@ bool pb_encode_tag(pb_ostream_t *stream, pb_wire_type_t wiretype, uint32_t field /* Encode an integer in the varint format. * This works for bool, enum, int32, int64, uint32 and uint64 field types. */ +#ifndef PB_WITHOUT_64BIT bool pb_encode_varint(pb_ostream_t *stream, uint64_t value); +#else +bool pb_encode_varint(pb_ostream_t *stream, uint32_t value); +#endif /* Encode an integer in the zig-zagged svarint format. * This works for sint32 and sint64. */ +#ifndef PB_WITHOUT_64BIT bool pb_encode_svarint(pb_ostream_t *stream, int64_t value); +#else +bool pb_encode_svarint(pb_ostream_t *stream, int32_t value); +#endif /* Encode a string or bytes type field. For strings, pass strlen(s) as size. */ -bool pb_encode_string(pb_ostream_t *stream, const uint8_t *buffer, size_t size); +bool pb_encode_string(pb_ostream_t *stream, const pb_byte_t *buffer, size_t size); /* Encode a fixed32, sfixed32 or float value. * You need to pass a pointer to a 4-byte wide C variable. */ bool pb_encode_fixed32(pb_ostream_t *stream, const void *value); +#ifndef PB_WITHOUT_64BIT /* Encode a fixed64, sfixed64 or double value. * You need to pass a pointer to a 8-byte wide C variable. */ bool pb_encode_fixed64(pb_ostream_t *stream, const void *value); +#endif /* Encode a submessage field. * You need to pass the pb_field_t array and pointer to struct, just like diff --git a/tests/Makefile b/tests/Makefile index 3c4e0b0..cee6bf6 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -10,12 +10,12 @@ coverage: # LCOV does not like the newer gcov format scons CC=gcc-4.6 CXX=gcc-4.6 - # We are only interested in pb_encode.o and pb_decode.o - find build -name '*.gcda' -and \! \( -name '*pb_encode*' -or -name '*pb_decode*' \) -exec rm '{}' \; - # Collect the data mkdir build/coverage lcov --base-directory . --directory build/ --gcov-tool gcov-4.6 -c -o build/coverage/nanopb.info + # Remove the test code from results + lcov -r build/coverage/nanopb.info '*tests*' -o build/coverage/nanopb.info + # Generate HTML genhtml -o build/coverage build/coverage/nanopb.info diff --git a/tests/SConstruct b/tests/SConstruct index 7a27844..f998024 100644 --- a/tests/SConstruct +++ b/tests/SConstruct @@ -57,6 +57,7 @@ if not env.GetOption('clean'): if not stdbool or not stdint or not stddef or not string: conf.env.Append(CPPDEFINES = {'PB_SYSTEM_HEADER': '\\"pb_syshdr.h\\"'}) conf.env.Append(CPPPATH = "#../extra") + conf.env.Append(SYSHDR = '\\"pb_syshdr.h\\"') if stdbool: conf.env.Append(CPPDEFINES = {'HAVE_STDBOOL_H': 1}) if stdint: conf.env.Append(CPPDEFINES = {'HAVE_STDINT_H': 1}) @@ -71,6 +72,11 @@ if not env.GetOption('clean'): else: conf.env.Append(PROTOCPATH = '/usr/include') + # Check protoc version + status, output = conf.TryAction('$PROTOC --version > $TARGET') + if status: + conf.env['PROTOC_VERSION'] = output + # Check if libmudflap is available (only with GCC) if 'gcc' in env['CC']: if conf.CheckLib('mudflap'): @@ -87,7 +93,9 @@ if not env.GetOption('clean'): conf.env.Append(CORECFLAGS = extra) # Check if we can use undefined behaviour sanitizer (only with clang) - extra = '-fsanitize=undefined ' + # TODO: Fuzz test triggers the bool sanitizer, figure out whether to + # modify the fuzz test or to keep ignoring the check. + extra = '-fsanitize=undefined,integer -fno-sanitize-recover=undefined,integer -fsanitize-recover=bool ' if 'clang' in env['CC']: if conf.CheckCCFLAGS(extra, linkflags = extra): conf.env.Append(CORECFLAGS = extra) @@ -101,7 +109,7 @@ if 'gcc' in env['CC']: # GNU Compiler Collection # Debug info, warnings as errors - env.Append(CFLAGS = '-ansi -pedantic -g -Wall -Werror -fprofile-arcs -ftest-coverage -fstack-protector-all') + env.Append(CFLAGS = '-ansi -pedantic -g -Wall -Werror -fprofile-arcs -ftest-coverage ') env.Append(CORECFLAGS = '-Wextra') env.Append(LINKFLAGS = '-g --coverage') @@ -120,9 +128,6 @@ elif 'cl' in env['CC']: # More strict checks on the nanopb core env.Append(CORECFLAGS = '/W4') - - # PB_RETURN_ERROR triggers C4127 because of while(0) - env.Append(CFLAGS = '/wd4127') elif 'tcc' in env['CC']: # Tiny C Compiler env.Append(CFLAGS = '-Wall -Werror -g') @@ -135,12 +140,18 @@ elif 'g++' in env['CXX'] or 'gcc' in env['CXX']: env.Append(CXXFLAGS = '-g -Wall -Werror -Wextra -Wno-missing-field-initializers') elif 'cl' in env['CXX']: env.Append(CXXFLAGS = '/Zi /W2 /WX') - + # Now include the SConscript files from all subdirectories import os.path env['VARIANT_DIR'] = 'build' env['BUILD'] = '#' + env['VARIANT_DIR'] env['COMMON'] = '#' + env['VARIANT_DIR'] + '/common' -for subdir in Glob('*/SConscript'): + +# Include common/SConscript first to make sure its exports are available +# to other SConscripts. +SConscript("common/SConscript", exports = 'env', variant_dir = env['VARIANT_DIR'] + '/common') + +for subdir in Glob('*/SConscript') + Glob('regression/*/SConscript'): + if str(subdir).startswith("common"): continue SConscript(subdir, exports = 'env', variant_dir = env['VARIANT_DIR'] + '/' + os.path.dirname(str(subdir))) diff --git a/tests/alltypes/SConscript b/tests/alltypes/SConscript index 9c9072b..13acd4e 100644 --- a/tests/alltypes/SConscript +++ b/tests/alltypes/SConscript @@ -4,8 +4,8 @@ Import("env") env.NanopbProto(["alltypes", "alltypes.options"]) -enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) # Test the round-trip from nanopb encoder to nanopb decoder env.RunTest(enc) @@ -32,4 +32,14 @@ env.Encode("optionals.output.recoded", MESSAGE='AllTypes') env.Compare(["optionals.output", "optionals.output.recoded"]) +# And for the _zero initializer +env.RunTest("zeroinit.output", enc, ARGS = ['2']) +env.RunTest("zeroinit.decout", [dec, "zeroinit.output"], ARGS = ['2']) +env.Decode("zeroinit.output.decoded", + ["zeroinit.output", "alltypes.proto"], + MESSAGE='AllTypes') +env.Encode("zeroinit.output.recoded", + ["zeroinit.output.decoded", "alltypes.proto"], + MESSAGE='AllTypes') +env.Compare(["zeroinit.output", "zeroinit.output.recoded"]) diff --git a/tests/alltypes/alltypes.options b/tests/alltypes/alltypes.options index b31e3cf..0d5ab12 100644 --- a/tests/alltypes/alltypes.options +++ b/tests/alltypes/alltypes.options @@ -1,3 +1,3 @@ * max_size:16 * max_count:5 - +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/alltypes/alltypes.proto b/tests/alltypes/alltypes.proto index db83c9a..2377180 100644 --- a/tests/alltypes/alltypes.proto +++ b/tests/alltypes/alltypes.proto @@ -1,3 +1,6 @@ +syntax = "proto2"; +// package name placeholder + message SubMessage { required string substuff1 = 1 [default = "1"]; required int32 substuff2 = 2 [default = 2]; @@ -55,7 +58,7 @@ message AllTypes { required SubMessage req_submsg = 16; required MyEnum req_enum = 17; required EmptyMessage req_emptymsg = 18; - + required bytes req_fbytes = 19; repeated int32 rep_int32 = 21 [packed = true]; repeated int64 rep_int64 = 22 [packed = true]; @@ -78,6 +81,7 @@ message AllTypes { repeated SubMessage rep_submsg = 36; repeated MyEnum rep_enum = 37 [packed = true]; repeated EmptyMessage rep_emptymsg = 38; + repeated bytes rep_fbytes = 39; optional int32 opt_int32 = 41 [default = 4041]; optional int64 opt_int64 = 42 [default = 4042]; @@ -96,11 +100,18 @@ message AllTypes { optional double opt_double = 53 [default = 4053]; optional string opt_string = 54 [default = "4054"]; - optional bytes opt_bytes = 55 [default = "4055"]; + optional bytes opt_bytes = 55 [default = "\x34\x5C\x00\xff"]; optional SubMessage opt_submsg = 56; optional MyEnum opt_enum = 57 [default = Second]; optional EmptyMessage opt_emptymsg = 58; + optional bytes opt_fbytes = 59 [default = "4059"]; + oneof oneof + { + SubMessage oneof_msg1 = 60; + EmptyMessage oneof_msg2 = 61; + } + // Check that extreme integer values are handled correctly required Limits req_limits = 98; diff --git a/tests/alltypes/decode_alltypes.c b/tests/alltypes/decode_alltypes.c index db72bb9..b74121f 100644 --- a/tests/alltypes/decode_alltypes.c +++ b/tests/alltypes/decode_alltypes.c @@ -19,65 +19,85 @@ the decoding and checks the fields. */ bool check_alltypes(pb_istream_t *stream, int mode) { - AllTypes alltypes; + /* Uses _init_default to just make sure that it works. */ + AllTypes alltypes = AllTypes_init_default; /* Fill with garbage to better detect initialization errors */ memset(&alltypes, 0xAA, sizeof(alltypes)); + alltypes.extensions = 0; if (!pb_decode(stream, AllTypes_fields, &alltypes)) return false; - TEST(alltypes.req_int32 == -1001); - TEST(alltypes.req_int64 == -1002); - TEST(alltypes.req_uint32 == 1003); - TEST(alltypes.req_uint64 == 1004); - TEST(alltypes.req_sint32 == -1005); - TEST(alltypes.req_sint64 == -1006); - TEST(alltypes.req_bool == true); - - TEST(alltypes.req_fixed32 == 1008); - TEST(alltypes.req_sfixed32 == -1009); - TEST(alltypes.req_float == 1010.0f); - - TEST(alltypes.req_fixed64 == 1011); - TEST(alltypes.req_sfixed64 == -1012); - TEST(alltypes.req_double == 1013.0f); - - TEST(strcmp(alltypes.req_string, "1014") == 0); - TEST(alltypes.req_bytes.size == 4); - TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0); - TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0); - TEST(alltypes.req_submsg.substuff2 == 1016); - TEST(alltypes.req_submsg.substuff3 == 3); - TEST(alltypes.req_enum == MyEnum_Truth); - - TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); - TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); - TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); - TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); - TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); - TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); - TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); - - TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); - TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); - TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); - - TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); - TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); - TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); - - TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); - TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); - TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + if (mode == 0 || mode == 1) + { + TEST(alltypes.req_int32 == -1001); + TEST(alltypes.req_int64 == -1002); + TEST(alltypes.req_uint32 == 1003); + TEST(alltypes.req_uint64 == 1004); + TEST(alltypes.req_sint32 == -1005); + TEST(alltypes.req_sint64 == -1006); + TEST(alltypes.req_bool == true); + + TEST(alltypes.req_fixed32 == 1008); + TEST(alltypes.req_sfixed32 == -1009); + TEST(alltypes.req_float == 1010.0f); + + TEST(alltypes.req_fixed64 == 1011); + TEST(alltypes.req_sfixed64 == -1012); + TEST(alltypes.req_double == 1013.0f); + + TEST(strcmp(alltypes.req_string, "1014") == 0); + TEST(alltypes.req_bytes.size == 4); + TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0); + TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0); + TEST(alltypes.req_submsg.substuff2 == 1016); + TEST(alltypes.req_submsg.substuff3 == 3); + TEST(alltypes.req_enum == MyEnum_Truth); + TEST(memcmp(alltypes.req_fbytes, "1019", 4) == 0); + + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); + TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); + TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); + TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); + + TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); + + TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); + TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); + TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); + + TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); + TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); - TEST(alltypes.rep_submsg_count == 5); - TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); - TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); - TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); - - TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); - TEST(alltypes.rep_emptymsg_count == 5); + TEST(alltypes.rep_submsg_count == 5); + TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); + TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); + + TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + TEST(alltypes.rep_emptymsg_count == 5); + TEST(alltypes.rep_fbytes_count == 5); + TEST(alltypes.rep_fbytes[0][0] == 0 && alltypes.rep_fbytes[0][3] == 0); + TEST(memcmp(alltypes.rep_fbytes[4], "2019", 4) == 0); + + TEST(alltypes.req_limits.int32_min == INT32_MIN); + TEST(alltypes.req_limits.int32_max == INT32_MAX); + TEST(alltypes.req_limits.uint32_min == 0); + TEST(alltypes.req_limits.uint32_max == UINT32_MAX); + TEST(alltypes.req_limits.int64_min == INT64_MIN); + TEST(alltypes.req_limits.int64_max == INT64_MAX); + TEST(alltypes.req_limits.uint64_min == 0); + TEST(alltypes.req_limits.uint64_max == UINT64_MAX); + TEST(alltypes.req_limits.enum_min == HugeEnum_Negative); + TEST(alltypes.req_limits.enum_max == HugeEnum_Positive); + } if (mode == 0) { @@ -115,7 +135,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(strcmp(alltypes.opt_string, "4054") == 0); TEST(alltypes.has_opt_bytes == false); TEST(alltypes.opt_bytes.size == 4); - TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0); + TEST(memcmp(alltypes.opt_bytes.bytes, "\x34\x5C\x00\xff", 4) == 0); TEST(alltypes.has_opt_submsg == false); TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0); TEST(alltypes.opt_submsg.substuff2 == 2); @@ -123,8 +143,12 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.has_opt_enum == false); TEST(alltypes.opt_enum == MyEnum_Second); TEST(alltypes.has_opt_emptymsg == false); + TEST(alltypes.has_opt_fbytes == false); + TEST(memcmp(alltypes.opt_fbytes, "4059", 4) == 0); + + TEST(alltypes.which_oneof == 0); } - else + else if (mode == 1) { /* Expect filled-in values */ TEST(alltypes.has_opt_int32 == true); @@ -168,18 +192,88 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.has_opt_enum == true); TEST(alltypes.opt_enum == MyEnum_Truth); TEST(alltypes.has_opt_emptymsg == true); + TEST(alltypes.has_opt_fbytes == true); + TEST(memcmp(alltypes.opt_fbytes, "3059", 4) == 0); + + TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); + TEST(strcmp(alltypes.oneof.oneof_msg1.substuff1, "4059") == 0); + TEST(alltypes.oneof.oneof_msg1.substuff2 == 4059); } + else if (mode == 2) + { + /* Expect zero values */ + TEST(alltypes.req_int32 == 0); + TEST(alltypes.req_int64 == 0); + TEST(alltypes.req_uint32 == 0); + TEST(alltypes.req_uint64 == 0); + TEST(alltypes.req_sint32 == 0); + TEST(alltypes.req_sint64 == 0); + TEST(alltypes.req_bool == false); + + TEST(alltypes.req_fixed32 == 0); + TEST(alltypes.req_sfixed32 == 0); + TEST(alltypes.req_float == 0.0f); + + TEST(alltypes.req_fixed64 == 0); + TEST(alltypes.req_sfixed64 == 0); + TEST(alltypes.req_double == 0.0f); + + TEST(strcmp(alltypes.req_string, "") == 0); + TEST(alltypes.req_bytes.size == 0); + TEST(strcmp(alltypes.req_submsg.substuff1, "") == 0); + TEST(alltypes.req_submsg.substuff2 == 0); + TEST(alltypes.req_enum == MyEnum_Zero); + + TEST(alltypes.rep_int32_count == 0); + TEST(alltypes.rep_int64_count == 0); + TEST(alltypes.rep_uint32_count == 0); + TEST(alltypes.rep_uint64_count == 0); + TEST(alltypes.rep_sint32_count == 0); + TEST(alltypes.rep_sint64_count == 0); + TEST(alltypes.rep_bool_count == 0); + + TEST(alltypes.rep_fixed32_count == 0); + TEST(alltypes.rep_sfixed32_count == 0); + TEST(alltypes.rep_float_count == 0); + + TEST(alltypes.rep_fixed64_count == 0); + TEST(alltypes.rep_sfixed64_count == 0); + TEST(alltypes.rep_double_count == 0); + + TEST(alltypes.rep_string_count == 0); + TEST(alltypes.rep_bytes_count == 0); + + TEST(alltypes.rep_submsg_count == 0); + + TEST(alltypes.rep_enum_count == 0); + TEST(alltypes.rep_emptymsg_count == 0); + TEST(alltypes.rep_fbytes_count == 0); - TEST(alltypes.req_limits.int32_min == INT32_MIN); - TEST(alltypes.req_limits.int32_max == INT32_MAX); - TEST(alltypes.req_limits.uint32_min == 0); - TEST(alltypes.req_limits.uint32_max == UINT32_MAX); - TEST(alltypes.req_limits.int64_min == INT64_MIN); - TEST(alltypes.req_limits.int64_max == INT64_MAX); - TEST(alltypes.req_limits.uint64_min == 0); - TEST(alltypes.req_limits.uint64_max == UINT64_MAX); - TEST(alltypes.req_limits.enum_min == HugeEnum_Negative); - TEST(alltypes.req_limits.enum_max == HugeEnum_Positive); + TEST(alltypes.has_opt_int32 == false); + TEST(alltypes.has_opt_int64 == false); + TEST(alltypes.has_opt_uint32 == false); + TEST(alltypes.has_opt_uint64 == false); + TEST(alltypes.has_opt_sint32 == false); + TEST(alltypes.has_opt_sint64 == false); + TEST(alltypes.has_opt_bool == false); + + TEST(alltypes.has_opt_fixed32 == false); + TEST(alltypes.has_opt_sfixed32 == false); + TEST(alltypes.has_opt_float == false); + + TEST(alltypes.has_opt_fixed64 == false); + TEST(alltypes.has_opt_sfixed64 == false); + TEST(alltypes.has_opt_double == false); + + TEST(alltypes.has_opt_string == false); + TEST(alltypes.has_opt_bytes == false); + TEST(alltypes.has_opt_submsg == false); + TEST(alltypes.has_opt_enum == false); + TEST(alltypes.has_opt_emptymsg == false); + TEST(alltypes.has_opt_fbytes == false); + + TEST(alltypes.which_oneof == 0); + } TEST(alltypes.end == 1099); diff --git a/tests/alltypes/encode_alltypes.c b/tests/alltypes/encode_alltypes.c index fa8eec9..15ea7b8 100644 --- a/tests/alltypes/encode_alltypes.c +++ b/tests/alltypes/encode_alltypes.c @@ -13,72 +13,79 @@ int main(int argc, char **argv) int mode = (argc > 1) ? atoi(argv[1]) : 0; /* Initialize the structure with constants */ - AllTypes alltypes = {0}; + AllTypes alltypes = AllTypes_init_zero; - alltypes.req_int32 = -1001; - alltypes.req_int64 = -1002; - alltypes.req_uint32 = 1003; - alltypes.req_uint64 = 1004; - alltypes.req_sint32 = -1005; - alltypes.req_sint64 = -1006; - alltypes.req_bool = true; - - alltypes.req_fixed32 = 1008; - alltypes.req_sfixed32 = -1009; - alltypes.req_float = 1010.0f; - - alltypes.req_fixed64 = 1011; - alltypes.req_sfixed64 = -1012; - alltypes.req_double = 1013.0; - - strcpy(alltypes.req_string, "1014"); - alltypes.req_bytes.size = 4; - memcpy(alltypes.req_bytes.bytes, "1015", 4); - strcpy(alltypes.req_submsg.substuff1, "1016"); - alltypes.req_submsg.substuff2 = 1016; - alltypes.req_enum = MyEnum_Truth; - - alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; - alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; - alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; - alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; - alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; - alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; - alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; - - alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; - alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; - alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; - - alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; - alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; - alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; - - alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); - alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; - memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); + if (mode == 0 || mode == 1) + { + alltypes.req_int32 = -1001; + alltypes.req_int64 = -1002; + alltypes.req_uint32 = 1003; + alltypes.req_uint64 = 1004; + alltypes.req_sint32 = -1005; + alltypes.req_sint64 = -1006; + alltypes.req_bool = true; + + alltypes.req_fixed32 = 1008; + alltypes.req_sfixed32 = -1009; + alltypes.req_float = 1010.0f; + + alltypes.req_fixed64 = 1011; + alltypes.req_sfixed64 = -1012; + alltypes.req_double = 1013.0; + + strcpy(alltypes.req_string, "1014"); + alltypes.req_bytes.size = 4; + memcpy(alltypes.req_bytes.bytes, "1015", 4); + strcpy(alltypes.req_submsg.substuff1, "1016"); + alltypes.req_submsg.substuff2 = 1016; + alltypes.req_enum = MyEnum_Truth; + memcpy(alltypes.req_fbytes, "1019", 4); + + alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; + alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; + alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; + alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; + alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; + alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; + + alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; + alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; + + alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; + alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; + alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; + + alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); + alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; + memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); - alltypes.rep_submsg_count = 5; - strcpy(alltypes.rep_submsg[4].substuff1, "2016"); - alltypes.rep_submsg[4].substuff2 = 2016; - alltypes.rep_submsg[4].has_substuff3 = true; - alltypes.rep_submsg[4].substuff3 = 2016; - - alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; - alltypes.rep_emptymsg_count = 5; - - alltypes.req_limits.int32_min = INT32_MIN; - alltypes.req_limits.int32_max = INT32_MAX; - alltypes.req_limits.uint32_min = 0; - alltypes.req_limits.uint32_max = UINT32_MAX; - alltypes.req_limits.int64_min = INT64_MIN; - alltypes.req_limits.int64_max = INT64_MAX; - alltypes.req_limits.uint64_min = 0; - alltypes.req_limits.uint64_max = UINT64_MAX; - alltypes.req_limits.enum_min = HugeEnum_Negative; - alltypes.req_limits.enum_max = HugeEnum_Positive; + alltypes.rep_submsg_count = 5; + strcpy(alltypes.rep_submsg[4].substuff1, "2016"); + alltypes.rep_submsg[4].substuff2 = 2016; + alltypes.rep_submsg[4].has_substuff3 = true; + alltypes.rep_submsg[4].substuff3 = 2016; + + alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + alltypes.rep_emptymsg_count = 5; + + alltypes.rep_fbytes_count = 5; + memcpy(alltypes.rep_fbytes[4], "2019", 4); + + alltypes.req_limits.int32_min = INT32_MIN; + alltypes.req_limits.int32_max = INT32_MAX; + alltypes.req_limits.uint32_min = 0; + alltypes.req_limits.uint32_max = UINT32_MAX; + alltypes.req_limits.int64_min = INT64_MIN; + alltypes.req_limits.int64_max = INT64_MAX; + alltypes.req_limits.uint64_min = 0; + alltypes.req_limits.uint64_max = UINT64_MAX; + alltypes.req_limits.enum_min = HugeEnum_Negative; + alltypes.req_limits.enum_max = HugeEnum_Positive; + } - if (mode != 0) + if (mode == 1) { /* Fill in values for optional fields */ alltypes.has_opt_int32 = true; @@ -121,6 +128,12 @@ int main(int argc, char **argv) alltypes.has_opt_enum = true; alltypes.opt_enum = MyEnum_Truth; alltypes.has_opt_emptymsg = true; + alltypes.has_opt_fbytes = true; + memcpy(alltypes.opt_fbytes, "3059", 4); + + alltypes.which_oneof = AllTypes_oneof_msg1_tag; + strcpy(alltypes.oneof.oneof_msg1.substuff1, "4059"); + alltypes.oneof.oneof_msg1.substuff2 = 4059; } alltypes.end = 1099; diff --git a/tests/alltypes_callback/SConscript b/tests/alltypes_callback/SConscript index 71b0160..8be5390 100644 --- a/tests/alltypes_callback/SConscript +++ b/tests/alltypes_callback/SConscript @@ -1,13 +1,13 @@ # Test the AllTypes encoding & decoding using callbacks for all fields. -Import("env") +Import("env", "malloc_env") c = Copy("$TARGET", "$SOURCE") env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) env.NanopbProto(["alltypes", "alltypes.options"]) -enc = env.Program(["encode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" @@ -21,3 +21,8 @@ env.RunTest("optionals.output", enc, ARGS = ['1']) env.RunTest("optionals.refdecout", [refdec, "optionals.output"], ARGS = ['1']) env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) +# Try with malloc support also +mallocbin1 = malloc_env.Object("decode_with_malloc.o", "decode_alltypes_callback.c") +mallocbin2 = malloc_env.Object("alltypes_malloc.pb.o", "alltypes.pb.c") +mallocdec = malloc_env.Program("decode_with_malloc", [mallocbin1, mallocbin2, "$COMMON/pb_decode_with_malloc.o", "$COMMON/pb_common_with_malloc.o", "$COMMON/malloc_wrappers.o"]) +env.RunTest("decode_with_malloc.output", [mallocdec, "encode_alltypes_callback.output"]) diff --git a/tests/alltypes_callback/alltypes.options b/tests/alltypes_callback/alltypes.options index a9c55ec..74d7a9c 100644 --- a/tests/alltypes_callback/alltypes.options +++ b/tests/alltypes_callback/alltypes.options @@ -1,3 +1,8 @@ # Generate all fields as callbacks. AllTypes.* type:FT_CALLBACK SubMessage.substuff1 max_size:16 +AllTypes.oneof no_unions:true + +# With FT_CALLBACK, these options should get ignored +*.*fbytes fixed_length:true max_size:4 + diff --git a/tests/alltypes_callback/decode_alltypes_callback.c b/tests/alltypes_callback/decode_alltypes_callback.c index 81274f6..576ce30 100644 --- a/tests/alltypes_callback/decode_alltypes_callback.c +++ b/tests/alltypes_callback/decode_alltypes_callback.c @@ -70,11 +70,15 @@ static bool read_string(pb_istream_t *stream, const pb_field_t *field, void **ar static bool read_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) { SubMessage submsg = {""}; + SubMessage *ref = *arg; if (!pb_decode(stream, SubMessage_fields, &submsg)) return false; - TEST(memcmp(&submsg, *arg, sizeof(submsg))); + TEST(strcmp(submsg.substuff1, ref->substuff1) == 0); + TEST(submsg.substuff2 == ref->substuff2); + TEST(submsg.has_substuff3 == ref->has_substuff3); + TEST(submsg.substuff3 == ref->substuff3); return true; } @@ -144,11 +148,16 @@ static bool read_repeated_string(pb_istream_t *stream, const pb_field_t *field, static bool read_repeated_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) { SubMessage** expected = (SubMessage**)arg; - SubMessage decoded = {""}; - if (!pb_decode(stream, SubMessage_fields, &decoded)) + SubMessage submsg = {""}; + if (!pb_decode(stream, SubMessage_fields, &submsg)) return false; - TEST(memcmp((*expected)++, &decoded, sizeof(decoded)) == 0); + TEST(strcmp(submsg.substuff1, (*expected)->substuff1) == 0); + TEST(submsg.substuff2 == (*expected)->substuff2); + TEST(submsg.has_substuff3 == (*expected)->has_substuff3); + TEST(submsg.substuff3 == (*expected)->substuff3); + (*expected)++; + return true; } @@ -177,13 +186,14 @@ static bool read_limits(pb_istream_t *stream, const pb_field_t *field, void **ar bool check_alltypes(pb_istream_t *stream, int mode) { /* Values for use from callbacks through pointers. */ + bool status; uint32_t req_fixed32 = 1008; int32_t req_sfixed32 = -1009; float req_float = 1010.0f; uint64_t req_fixed64 = 1011; int64_t req_sfixed64 = -1012; double req_double = 1013.0; - SubMessage req_submsg = {"1016", 1016}; + SubMessage req_submsg = {"1016", 1016, false, 3}; int32_t rep_int32[5] = {0, 0, 0, 0, -2001}; int32_t rep_int64[5] = {0, 0, 0, 0, -2002}; @@ -213,13 +223,12 @@ bool check_alltypes(pb_istream_t *stream, int mode) uint64_t opt_fixed64 = 3051; int64_t opt_sfixed64 = 3052; double opt_double = 3053.0f; - SubMessage opt_submsg = {"3056", 3056}; + SubMessage opt_submsg = {"3056", 3056, false, 3}; + + SubMessage oneof_msg1 = {"4059", 4059, false, 3}; /* Bind callbacks for required fields */ - AllTypes alltypes; - - /* Fill with garbage to better detect initialization errors */ - memset(&alltypes, 0xAA, sizeof(alltypes)); + AllTypes alltypes = AllTypes_init_zero; alltypes.req_int32.funcs.decode = &read_varint; alltypes.req_int32.arg = (void*)-1001; @@ -391,9 +400,19 @@ bool check_alltypes(pb_istream_t *stream, int mode) alltypes.opt_enum.arg = (void*)MyEnum_Truth; alltypes.opt_emptymsg.funcs.decode = &read_emptymsg; + + alltypes.oneof_msg1.funcs.decode = &read_submsg; + alltypes.oneof_msg1.arg = &oneof_msg1; } - return pb_decode(stream, AllTypes_fields, &alltypes); + status = pb_decode(stream, AllTypes_fields, &alltypes); + +#ifdef PB_ENABLE_MALLOC + /* Just to check for any interference between pb_release() and callback fields */ + pb_release(AllTypes_fields, &alltypes); +#endif + + return status; } int main(int argc, char **argv) diff --git a/tests/alltypes_callback/encode_alltypes_callback.c b/tests/alltypes_callback/encode_alltypes_callback.c index 10560b1..b206783 100644 --- a/tests/alltypes_callback/encode_alltypes_callback.c +++ b/tests/alltypes_callback/encode_alltypes_callback.c @@ -202,6 +202,8 @@ int main(int argc, char **argv) double opt_double = 3053.0f; SubMessage opt_submsg = {"3056", 3056}; + SubMessage oneof_msg1 = {"4059", 4059}; + /* Bind callbacks for required fields */ AllTypes alltypes = {{{0}}}; @@ -261,6 +263,9 @@ int main(int argc, char **argv) alltypes.req_emptymsg.funcs.encode = &write_emptymsg; + alltypes.req_fbytes.funcs.encode = &write_string; + alltypes.req_fbytes.arg = "1019"; + /* Bind callbacks for repeated fields */ alltypes.rep_int32.funcs.encode = &write_repeated_varint; alltypes.rep_int32.arg = (void*)-2001; @@ -315,6 +320,9 @@ int main(int argc, char **argv) alltypes.rep_emptymsg.funcs.encode = &write_repeated_emptymsg; + alltypes.rep_fbytes.funcs.encode = &write_repeated_string; + alltypes.rep_fbytes.arg = "2019"; + alltypes.req_limits.funcs.encode = &write_limits; /* Bind callbacks for optional fields */ @@ -372,6 +380,12 @@ int main(int argc, char **argv) alltypes.opt_enum.arg = (void*)MyEnum_Truth; alltypes.opt_emptymsg.funcs.encode = &write_emptymsg; + + alltypes.opt_fbytes.funcs.encode = &write_string; + alltypes.opt_fbytes.arg = "3059"; + + alltypes.oneof_msg1.funcs.encode = &write_submsg; + alltypes.oneof_msg1.arg = &oneof_msg1; } alltypes.end.funcs.encode = &write_varint; diff --git a/tests/alltypes_pointer/SConscript b/tests/alltypes_pointer/SConscript index e48d6aa..b095ae0 100644 --- a/tests/alltypes_pointer/SConscript +++ b/tests/alltypes_pointer/SConscript @@ -1,30 +1,22 @@ # Encode the AllTypes message using pointers for all fields, and verify the # output against the normal AllTypes test case. -Import("env") - -# We need our own pb_decode.o for the malloc support -env = env.Clone() -env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1}); - -# Disable libmudflap, because it will confuse valgrind -# and other memory leak detection tools. -if '-fmudflap' in env["CCFLAGS"]: - env["CCFLAGS"].remove("-fmudflap") - env["LINKFLAGS"].remove("-fmudflap") - env["LIBS"].remove("mudflap") - -strict = env.Clone() -strict.Append(CFLAGS = strict['CORECFLAGS']) -strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") -strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c") +Import("env", "malloc_env") c = Copy("$TARGET", "$SOURCE") env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) env.NanopbProto(["alltypes", "alltypes.options"]) -enc = env.Program(["encode_alltypes_pointer.c", "alltypes.pb.c", "pb_encode_with_malloc.o"]) -dec = env.Program(["decode_alltypes_pointer.c", "alltypes.pb.c", "pb_decode_with_malloc.o"]) +enc = malloc_env.Program(["encode_alltypes_pointer.c", + "alltypes.pb.c", + "$COMMON/pb_encode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) +dec = malloc_env.Program(["decode_alltypes_pointer.c", + "alltypes.pb.c", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) # Encode and compare results to non-pointer alltypes test case env.RunTest(enc) @@ -35,7 +27,7 @@ valgrind = env.WhereIs('valgrind') kwargs = {} if valgrind: kwargs['COMMAND'] = valgrind - kwargs['ARGS'] = ["-q", dec[0].abspath] + kwargs['ARGS'] = ["-q", "--error-exitcode=99", dec[0].abspath] env.RunTest("decode_alltypes.output", [dec, "encode_alltypes_pointer.output"], **kwargs) diff --git a/tests/alltypes_pointer/alltypes.options b/tests/alltypes_pointer/alltypes.options index 52abeb7..8699fe2 100644 --- a/tests/alltypes_pointer/alltypes.options +++ b/tests/alltypes_pointer/alltypes.options @@ -1,3 +1,4 @@ # Generate all fields as pointers. * type:FT_POINTER +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/alltypes_pointer/decode_alltypes_pointer.c b/tests/alltypes_pointer/decode_alltypes_pointer.c index 889676b..4ee6f8b 100644 --- a/tests/alltypes_pointer/decode_alltypes_pointer.c +++ b/tests/alltypes_pointer/decode_alltypes_pointer.c @@ -19,6 +19,7 @@ bool check_alltypes(pb_istream_t *stream, int mode) /* Fill with garbage to better detect initialization errors */ memset(&alltypes, 0xAA, sizeof(alltypes)); + alltypes.extensions = 0; if (!pb_decode(stream, AllTypes_fields, &alltypes)) return false; @@ -46,7 +47,8 @@ bool check_alltypes(pb_istream_t *stream, int mode) && strcmp(alltypes.req_submsg->substuff1, "1016") == 0); TEST(alltypes.req_submsg && alltypes.req_submsg->substuff2 && *alltypes.req_submsg->substuff2 == 1016); - TEST(*alltypes.req_enum == MyEnum_Truth); + TEST(alltypes.req_enum && *alltypes.req_enum == MyEnum_Truth); + TEST(alltypes.req_fbytes && memcmp(alltypes.req_fbytes, "1019", 4) == 0); TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); @@ -75,6 +77,9 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); TEST(alltypes.rep_emptymsg_count == 5); + TEST(alltypes.rep_fbytes_count == 5); + TEST(alltypes.rep_fbytes[0][0] == 0 && alltypes.rep_fbytes[0][3] == 0); + TEST(memcmp(alltypes.rep_fbytes[4], "2019", 4) == 0); if (mode == 0) { @@ -98,6 +103,9 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.opt_bytes == NULL); TEST(alltypes.opt_submsg == NULL); TEST(alltypes.opt_enum == NULL); + TEST(alltypes.opt_fbytes == NULL); + + TEST(alltypes.which_oneof == 0); } else { @@ -124,6 +132,11 @@ bool check_alltypes(pb_istream_t *stream, int mode) TEST(alltypes.opt_submsg && *alltypes.opt_submsg->substuff2 == 3056); TEST(alltypes.opt_enum && *alltypes.opt_enum == MyEnum_Truth); TEST(alltypes.opt_emptymsg); + TEST(alltypes.opt_fbytes && memcmp(alltypes.opt_fbytes, "3059", 4) == 0); + + TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); + TEST(alltypes.oneof.oneof_msg1 && strcmp(alltypes.oneof.oneof_msg1->substuff1, "4059") == 0); + TEST(alltypes.oneof.oneof_msg1->substuff2 && *alltypes.oneof.oneof_msg1->substuff2 == 4059); } TEST(alltypes.req_limits->int32_min && *alltypes.req_limits->int32_min == INT32_MIN); diff --git a/tests/alltypes_pointer/encode_alltypes_pointer.c b/tests/alltypes_pointer/encode_alltypes_pointer.c index c128569..a39af6f 100644 --- a/tests/alltypes_pointer/encode_alltypes_pointer.c +++ b/tests/alltypes_pointer/encode_alltypes_pointer.c @@ -32,6 +32,7 @@ int main(int argc, char **argv) SubMessage req_submsg = {"1016", &req_substuff}; MyEnum req_enum = MyEnum_Truth; EmptyMessage req_emptymsg = {0}; + pb_byte_t req_fbytes[4] = {'1', '0', '1', '9'}; int32_t end = 1099; @@ -62,6 +63,7 @@ int main(int argc, char **argv) {"2016", &rep_substuff2, &rep_substuff3}}; MyEnum rep_enum[5] = {0, 0, 0, 0, MyEnum_Truth}; EmptyMessage rep_emptymsg[5] = {{0}, {0}, {0}, {0}, {0}}; + pb_byte_t rep_fbytes[5][4] = {{0}, {0}, {0}, {0}, {'2', '0', '1', '9'}}; /* Values for optional fields */ int32_t opt_int32 = 3041; @@ -83,6 +85,10 @@ int main(int argc, char **argv) SubMessage opt_submsg = {"3056", &opt_substuff}; MyEnum opt_enum = MyEnum_Truth; EmptyMessage opt_emptymsg = {0}; + pb_byte_t opt_fbytes[4] = {'3', '0', '5', '9'}; + + static int32_t oneof_substuff = 4059; + SubMessage oneof_msg1 = {"4059", &oneof_substuff}; /* Values for the Limits message. */ static int32_t int32_min = INT32_MIN; @@ -122,6 +128,7 @@ int main(int argc, char **argv) alltypes.req_submsg = &req_submsg; alltypes.req_enum = &req_enum; alltypes.req_emptymsg = &req_emptymsg; + alltypes.req_fbytes = &req_fbytes; alltypes.req_limits = &req_limits; alltypes.rep_int32_count = 5; alltypes.rep_int32 = rep_int32; @@ -142,6 +149,7 @@ int main(int argc, char **argv) alltypes.rep_submsg_count = 5; alltypes.rep_submsg = rep_submsg; alltypes.rep_enum_count = 5; alltypes.rep_enum = rep_enum; alltypes.rep_emptymsg_count = 5; alltypes.rep_emptymsg = rep_emptymsg; + alltypes.rep_fbytes_count = 5; alltypes.rep_fbytes = rep_fbytes; if (mode != 0) { @@ -164,6 +172,10 @@ int main(int argc, char **argv) alltypes.opt_submsg = &opt_submsg; alltypes.opt_enum = &opt_enum; alltypes.opt_emptymsg = &opt_emptymsg; + alltypes.opt_fbytes = &opt_fbytes; + + alltypes.which_oneof = AllTypes_oneof_msg1_tag; + alltypes.oneof.oneof_msg1 = &oneof_msg1; } alltypes.end = &end; diff --git a/tests/alltypes_proto3/SConscript b/tests/alltypes_proto3/SConscript new file mode 100644 index 0000000..4c2388e --- /dev/null +++ b/tests/alltypes_proto3/SConscript @@ -0,0 +1,45 @@ +# Version of AllTypes test case for protobuf 3 file format. + +Import("env") + +import re +match = None +if 'PROTOC_VERSION' in env: + match = re.search('([0-9]+).([0-9]+).([0-9]+)', env['PROTOC_VERSION']) + +if match: + version = list(map(int, match.groups())) + +# proto3 syntax is supported by protoc >= 3.0.0 +if env.GetOption('clean') or (match and version[0] >= 3): + + env.NanopbProto(["alltypes", "alltypes.options"]) + enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) + dec = env.Program(["decode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) + + # Test the round-trip from nanopb encoder to nanopb decoder + env.RunTest(enc) + env.RunTest([dec, "encode_alltypes.output"]) + + # Re-encode the data using protoc, and check that the results from nanopb + # match byte-per-byte to the protoc output. + env.Decode("encode_alltypes.output.decoded", + ["encode_alltypes.output", "alltypes.proto"], + MESSAGE='AllTypes') + env.Encode("encode_alltypes.output.recoded", + ["encode_alltypes.output.decoded", "alltypes.proto"], + MESSAGE='AllTypes') + env.Compare(["encode_alltypes.output", "encode_alltypes.output.recoded"]) + + # Do the same checks with the optional fields present. + env.RunTest("optionals.output", enc, ARGS = ['1']) + env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) + env.Decode("optionals.output.decoded", + ["optionals.output", "alltypes.proto"], + MESSAGE='AllTypes') + env.Encode("optionals.output.recoded", + ["optionals.output.decoded", "alltypes.proto"], + MESSAGE='AllTypes') + env.Compare(["optionals.output", "optionals.output.recoded"]) + + diff --git a/tests/alltypes_proto3/alltypes.options b/tests/alltypes_proto3/alltypes.options new file mode 100644 index 0000000..78dd08d --- /dev/null +++ b/tests/alltypes_proto3/alltypes.options @@ -0,0 +1,4 @@ +* max_size:16 +* max_count:5 +*.*fbytes fixed_length:true max_size:4 + diff --git a/tests/alltypes_proto3/alltypes.proto b/tests/alltypes_proto3/alltypes.proto new file mode 100644 index 0000000..f66109e --- /dev/null +++ b/tests/alltypes_proto3/alltypes.proto @@ -0,0 +1,100 @@ +syntax = "proto3"; +// package name placeholder + +message SubMessage { + string substuff1 = 1; + int32 substuff2 = 2; + fixed32 substuff3 = 3; +} + +message EmptyMessage { + +} + +enum HugeEnum { + HE_Zero = 0; + Negative = -2147483647; /* protoc doesn't accept -2147483648 here */ + Positive = 2147483647; +} + +message Limits { + int32 int32_min = 1; + int32 int32_max = 2; + uint32 uint32_min = 3; + uint32 uint32_max = 4; + int64 int64_min = 5; + int64 int64_max = 6; + uint64 uint64_min = 7; + uint64 uint64_max = 8; + HugeEnum enum_min = 9; + HugeEnum enum_max = 10; +} + +enum MyEnum { + Zero = 0; + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + int32 sng_int32 = 1; + int64 sng_int64 = 2; + uint32 sng_uint32 = 3; + uint64 sng_uint64 = 4; + sint32 sng_sint32 = 5; + sint64 sng_sint64 = 6; + bool sng_bool = 7; + + fixed32 sng_fixed32 = 8; + sfixed32 sng_sfixed32= 9; + float sng_float = 10; + + fixed64 sng_fixed64 = 11; + sfixed64 sng_sfixed64= 12; + double sng_double = 13; + + string sng_string = 14; + bytes sng_bytes = 15; + SubMessage sng_submsg = 16; + MyEnum sng_enum = 17; + EmptyMessage sng_emptymsg = 18; + bytes sng_fbytes = 19; + + repeated int32 rep_int32 = 21 [packed = true]; + repeated int64 rep_int64 = 22 [packed = true]; + repeated uint32 rep_uint32 = 23 [packed = true]; + repeated uint64 rep_uint64 = 24 [packed = true]; + repeated sint32 rep_sint32 = 25 [packed = true]; + repeated sint64 rep_sint64 = 26 [packed = true]; + repeated bool rep_bool = 27 [packed = true]; + + repeated fixed32 rep_fixed32 = 28 [packed = true]; + repeated sfixed32 rep_sfixed32= 29 [packed = true]; + repeated float rep_float = 30 [packed = true]; + + repeated fixed64 rep_fixed64 = 31 [packed = true]; + repeated sfixed64 rep_sfixed64= 32 [packed = true]; + repeated double rep_double = 33 [packed = true]; + + repeated string rep_string = 34; + repeated bytes rep_bytes = 35; + repeated SubMessage rep_submsg = 36; + repeated MyEnum rep_enum = 37 [packed = true]; + repeated EmptyMessage rep_emptymsg = 38; + repeated bytes rep_fbytes = 39; + + oneof oneof + { + SubMessage oneof_msg1 = 59; + EmptyMessage oneof_msg2 = 60; + } + + // Check that extreme integer values are handled correctly + Limits req_limits = 98; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + int32 end = 99; +} + diff --git a/tests/alltypes_proto3/decode_alltypes.c b/tests/alltypes_proto3/decode_alltypes.c new file mode 100644 index 0000000..51c1c41 --- /dev/null +++ b/tests/alltypes_proto3/decode_alltypes.c @@ -0,0 +1,167 @@ +/* Tests the decoding of all types. + * This is the counterpart of test_encode3. + * Run e.g. ./test_encode3 | ./test_decode3 + */ + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <pb_decode.h> +#include "alltypes.pb.h" +#include "test_helpers.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return false; \ + } + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + AllTypes alltypes = AllTypes_init_zero; + + /* Fill with garbage to better detect initialization errors */ + memset(&alltypes, 0xAA, sizeof(alltypes)); + + if (!pb_decode(stream, AllTypes_fields, &alltypes)) + return false; + + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); + TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); + TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); + TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); + + TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); + + TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); + TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); + TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); + + TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); + TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + + TEST(alltypes.rep_submsg_count == 5); + TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); + TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 0); + + TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + TEST(alltypes.rep_emptymsg_count == 5); + + TEST(alltypes.rep_fbytes_count == 5); + TEST(alltypes.rep_fbytes[0][0] == 0 && alltypes.rep_fbytes[0][3] == 0); + TEST(memcmp(alltypes.rep_fbytes[4], "2019", 4) == 0); + + if (mode == 0) + { + /* Expect default values */ + TEST(alltypes.sng_int32 == 0); + TEST(alltypes.sng_int64 == 0); + TEST(alltypes.sng_uint32 == 0); + TEST(alltypes.sng_uint64 == 0); + TEST(alltypes.sng_sint32 == 0); + TEST(alltypes.sng_sint64 == 0); + TEST(alltypes.sng_bool == false); + + TEST(alltypes.sng_fixed32 == 0); + TEST(alltypes.sng_sfixed32 == 0); + TEST(alltypes.sng_float == 0.0f); + + TEST(alltypes.sng_fixed64 == 0); + TEST(alltypes.sng_sfixed64 == 0); + TEST(alltypes.sng_double == 0.0); + + TEST(strcmp(alltypes.sng_string, "") == 0); + TEST(alltypes.sng_bytes.size == 0); + TEST(strcmp(alltypes.sng_submsg.substuff1, "") == 0); + TEST(alltypes.sng_submsg.substuff2 == 0); + TEST(alltypes.sng_submsg.substuff3 == 0); + TEST(alltypes.sng_enum == MyEnum_Zero); + TEST(alltypes.sng_fbytes[0] == 0 && + alltypes.sng_fbytes[1] == 0 && + alltypes.sng_fbytes[2] == 0 && + alltypes.sng_fbytes[3] == 0); + + TEST(alltypes.which_oneof == 0); + } + else + { + /* Expect filled-in values */ + TEST(alltypes.sng_int32 == 3041); + TEST(alltypes.sng_int64 == 3042); + TEST(alltypes.sng_uint32 == 3043); + TEST(alltypes.sng_uint64 == 3044); + TEST(alltypes.sng_sint32 == 3045); + TEST(alltypes.sng_sint64 == 3046); + TEST(alltypes.sng_bool == true); + + TEST(alltypes.sng_fixed32 == 3048); + TEST(alltypes.sng_sfixed32 == 3049); + TEST(alltypes.sng_float == 3050.0f); + + TEST(alltypes.sng_fixed64 == 3051); + TEST(alltypes.sng_sfixed64 == 3052); + TEST(alltypes.sng_double == 3053.0); + + TEST(strcmp(alltypes.sng_string, "3054") == 0); + TEST(alltypes.sng_bytes.size == 4); + TEST(memcmp(alltypes.sng_bytes.bytes, "3055", 4) == 0); + TEST(strcmp(alltypes.sng_submsg.substuff1, "3056") == 0); + TEST(alltypes.sng_submsg.substuff2 == 3056); + TEST(alltypes.sng_submsg.substuff3 == 0); + TEST(alltypes.sng_enum == MyEnum_Truth); + TEST(memcmp(alltypes.sng_fbytes, "3059", 4) == 0); + + TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); + TEST(strcmp(alltypes.oneof.oneof_msg1.substuff1, "4059") == 0); + TEST(alltypes.oneof.oneof_msg1.substuff2 == 4059); + } + + TEST(alltypes.req_limits.int32_min == INT32_MIN); + TEST(alltypes.req_limits.int32_max == INT32_MAX); + TEST(alltypes.req_limits.uint32_min == 0); + TEST(alltypes.req_limits.uint32_max == UINT32_MAX); + TEST(alltypes.req_limits.int64_min == INT64_MIN); + TEST(alltypes.req_limits.int64_max == INT64_MAX); + TEST(alltypes.req_limits.uint64_min == 0); + TEST(alltypes.req_limits.uint64_max == UINT64_MAX); + TEST(alltypes.req_limits.enum_min == HugeEnum_Negative); + TEST(alltypes.req_limits.enum_max == HugeEnum_Positive); + + TEST(alltypes.end == 1099); + + return true; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[1024]; + size_t count; + pb_istream_t stream; + + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/alltypes_proto3/encode_alltypes.c b/tests/alltypes_proto3/encode_alltypes.c new file mode 100644 index 0000000..1da0668 --- /dev/null +++ b/tests/alltypes_proto3/encode_alltypes.c @@ -0,0 +1,111 @@ +/* Attempts to test all the datatypes supported by ProtoBuf3. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pb_encode.h> +#include "alltypes.pb.h" +#include "test_helpers.h" + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Initialize the structure with constants */ + AllTypes alltypes = AllTypes_init_zero; + + alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; + alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; + alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; + alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; + alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; + alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; + + alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; + alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; + + alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; + alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; + alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; + + alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); + alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; + memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); + + alltypes.rep_submsg_count = 5; + strcpy(alltypes.rep_submsg[4].substuff1, "2016"); + alltypes.rep_submsg[4].substuff2 = 2016; + alltypes.rep_submsg[4].substuff3 = 2016; + + alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + alltypes.rep_emptymsg_count = 5; + + alltypes.rep_fbytes_count = 5; + memcpy(alltypes.rep_fbytes[4], "2019", 4); + + alltypes.req_limits.int32_min = INT32_MIN; + alltypes.req_limits.int32_max = INT32_MAX; + alltypes.req_limits.uint32_min = 0; + alltypes.req_limits.uint32_max = UINT32_MAX; + alltypes.req_limits.int64_min = INT64_MIN; + alltypes.req_limits.int64_max = INT64_MAX; + alltypes.req_limits.uint64_min = 0; + alltypes.req_limits.uint64_max = UINT64_MAX; + alltypes.req_limits.enum_min = HugeEnum_Negative; + alltypes.req_limits.enum_max = HugeEnum_Positive; + + if (mode != 0) + { + /* Fill in values for singular fields */ + alltypes.sng_int32 = 3041; + alltypes.sng_int64 = 3042; + alltypes.sng_uint32 = 3043; + alltypes.sng_uint64 = 3044; + alltypes.sng_sint32 = 3045; + alltypes.sng_sint64 = 3046; + alltypes.sng_bool = true; + + alltypes.sng_fixed32 = 3048; + alltypes.sng_sfixed32 = 3049; + alltypes.sng_float = 3050.0f; + + alltypes.sng_fixed64 = 3051; + alltypes.sng_sfixed64 = 3052; + alltypes.sng_double = 3053.0; + + strcpy(alltypes.sng_string, "3054"); + alltypes.sng_bytes.size = 4; + memcpy(alltypes.sng_bytes.bytes, "3055", 4); + strcpy(alltypes.sng_submsg.substuff1, "3056"); + alltypes.sng_submsg.substuff2 = 3056; + alltypes.sng_enum = MyEnum_Truth; + memcpy(alltypes.sng_fbytes, "3059", 4); + + alltypes.which_oneof = AllTypes_oneof_msg1_tag; + strcpy(alltypes.oneof.oneof_msg1.substuff1, "4059"); + alltypes.oneof.oneof_msg1.substuff2 = 4059; + } + + alltypes.end = 1099; + + { + uint8_t buffer[AllTypes_size]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } + } +} diff --git a/tests/alltypes_proto3_callback/SConscript b/tests/alltypes_proto3_callback/SConscript new file mode 100644 index 0000000..183a138 --- /dev/null +++ b/tests/alltypes_proto3_callback/SConscript @@ -0,0 +1,23 @@ +# Test the AllTypes encoding & decoding using callbacks for all fields. + +Import("env", "malloc_env") + +c = Copy("$TARGET", "$SOURCE") +env.Command("alltypes.proto", "#alltypes_proto3/alltypes.proto", c) + +env.NanopbProto(["alltypes", "alltypes.options"]) +enc = env.Program(["encode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_alltypes_callback.c", "alltypes.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) + +refdec = "$BUILD/alltypes_proto3/decode_alltypes$PROGSUFFIX" + +# Encode and compare results +env.RunTest(enc) +env.RunTest("decode_alltypes.output", [refdec, "encode_alltypes_callback.output"]) +env.RunTest("decode_alltypes_callback.output", [dec, "encode_alltypes_callback.output"]) + +# Do the same thing with the optional fields present +env.RunTest("optionals.output", enc, ARGS = ['1']) +env.RunTest("optionals.refdecout", [refdec, "optionals.output"], ARGS = ['1']) +env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) + diff --git a/tests/alltypes_proto3_callback/alltypes.options b/tests/alltypes_proto3_callback/alltypes.options new file mode 100644 index 0000000..74d7a9c --- /dev/null +++ b/tests/alltypes_proto3_callback/alltypes.options @@ -0,0 +1,8 @@ +# Generate all fields as callbacks. +AllTypes.* type:FT_CALLBACK +SubMessage.substuff1 max_size:16 +AllTypes.oneof no_unions:true + +# With FT_CALLBACK, these options should get ignored +*.*fbytes fixed_length:true max_size:4 + diff --git a/tests/alltypes_proto3_callback/decode_alltypes_callback.c b/tests/alltypes_proto3_callback/decode_alltypes_callback.c new file mode 100644 index 0000000..2b3c2f3 --- /dev/null +++ b/tests/alltypes_proto3_callback/decode_alltypes_callback.c @@ -0,0 +1,376 @@ +/* Attempts to test all the datatypes supported by ProtoBuf when used as callback fields. + * Note that normally there would be no reason to use callback fields for this, + * because each encoder defined here only gives a single field. + */ + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <pb_decode.h> +#include "alltypes.pb.h" +#include "test_helpers.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed (in field %d).\n", field->tag); \ + return false; \ + } + +static bool read_varint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint64_t value; + if (!pb_decode_varint(stream, &value)) + return false; + + TEST((int64_t)value == (long)*arg); + return true; +} + +static bool read_svarint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + int64_t value; + if (!pb_decode_svarint(stream, &value)) + return false; + + TEST(value == (long)*arg); + return true; +} + +static bool read_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint32_t value; + if (!pb_decode_fixed32(stream, &value)) + return false; + + TEST(value == *(uint32_t*)*arg); + return true; +} + +static bool read_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint64_t value; + if (!pb_decode_fixed64(stream, &value)) + return false; + + TEST(value == *(uint64_t*)*arg); + return true; +} + +static bool read_string(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint8_t buf[16] = {0}; + size_t len = stream->bytes_left; + + if (len > sizeof(buf) - 1 || !pb_read(stream, buf, len)) + return false; + + TEST(strcmp((char*)buf, *arg) == 0); + return true; +} + +static bool read_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + SubMessage submsg = {""}; + SubMessage *ref = *arg; + + if (!pb_decode(stream, SubMessage_fields, &submsg)) + return false; + + TEST(strcmp(submsg.substuff1, ref->substuff1) == 0); + TEST(submsg.substuff2 == ref->substuff2); + TEST(submsg.substuff3 == ref->substuff3); + return true; +} + +static bool read_emptymsg(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + EmptyMessage emptymsg = {0}; + return pb_decode(stream, EmptyMessage_fields, &emptymsg); +} + +static bool read_repeated_varint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + int32_t** expected = (int32_t**)arg; + uint64_t value; + if (!pb_decode_varint(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_svarint(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + int32_t** expected = (int32_t**)arg; + int64_t value; + if (!pb_decode_svarint(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_fixed32(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint32_t** expected = (uint32_t**)arg; + uint32_t value; + if (!pb_decode_fixed32(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_fixed64(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint64_t** expected = (uint64_t**)arg; + uint64_t value; + if (!pb_decode_fixed64(stream, &value)) + return false; + + TEST(*(*expected)++ == value); + return true; +} + +static bool read_repeated_string(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint8_t*** expected = (uint8_t***)arg; + uint8_t buf[16] = {0}; + size_t len = stream->bytes_left; + + if (len > sizeof(buf) - 1 || !pb_read(stream, buf, len)) + return false; + + TEST(strcmp((char*)*(*expected)++, (char*)buf) == 0); + return true; +} + +static bool read_repeated_submsg(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + SubMessage** expected = (SubMessage**)arg; + SubMessage submsg = {""}; + if (!pb_decode(stream, SubMessage_fields, &submsg)) + return false; + + TEST(strcmp(submsg.substuff1, (*expected)->substuff1) == 0); + TEST(submsg.substuff2 == (*expected)->substuff2); + TEST(submsg.substuff3 == (*expected)->substuff3); + (*expected)++; + + return true; +} + +static bool read_limits(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + Limits decoded = {0}; + if (!pb_decode(stream, Limits_fields, &decoded)) + return false; + + TEST(decoded.int32_min == INT32_MIN); + TEST(decoded.int32_max == INT32_MAX); + TEST(decoded.uint32_min == 0); + TEST(decoded.uint32_max == UINT32_MAX); + TEST(decoded.int64_min == INT64_MIN); + TEST(decoded.int64_max == INT64_MAX); + TEST(decoded.uint64_min == 0); + TEST(decoded.uint64_max == UINT64_MAX); + TEST(decoded.enum_min == HugeEnum_Negative); + TEST(decoded.enum_max == HugeEnum_Positive); + + return true; +} + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + /* Values for use from callbacks through pointers. */ + bool status; + + int32_t rep_int32[5] = {0, 0, 0, 0, -2001}; + int32_t rep_int64[5] = {0, 0, 0, 0, -2002}; + int32_t rep_uint32[5] = {0, 0, 0, 0, 2003}; + int32_t rep_uint64[5] = {0, 0, 0, 0, 2004}; + int32_t rep_sint32[5] = {0, 0, 0, 0, -2005}; + int32_t rep_sint64[5] = {0, 0, 0, 0, -2006}; + int32_t rep_bool[5] = {false, false, false, false, true}; + uint32_t rep_fixed32[5] = {0, 0, 0, 0, 2008}; + int32_t rep_sfixed32[5] = {0, 0, 0, 0, -2009}; + float rep_float[5] = {0, 0, 0, 0, 2010.0f}; + uint64_t rep_fixed64[5] = {0, 0, 0, 0, 2011}; + int64_t rep_sfixed64[5] = {0, 0, 0, 0, -2012}; + double rep_double[5] = {0, 0, 0, 0, 2013.0}; + char* rep_string[5] = {"", "", "", "", "2014"}; + char* rep_bytes[5] = {"", "", "", "", "2015"}; + SubMessage rep_submsg[5] = {{"", 0, 0}, + {"", 0, 0}, + {"", 0, 0}, + {"", 0, 0}, + {"2016", 2016, 2016}}; + int32_t rep_enum[5] = {0, 0, 0, 0, MyEnum_Truth}; + + uint32_t sng_fixed32 = 3048; + int32_t sng_sfixed32 = 3049; + float sng_float = 3050.0f; + uint64_t sng_fixed64 = 3051; + int64_t sng_sfixed64 = 3052; + double sng_double = 3053.0f; + SubMessage sng_submsg = {"3056", 3056}; + + SubMessage oneof_msg1 = {"4059", 4059}; + + AllTypes alltypes = AllTypes_init_zero; + + /* Bind callbacks for repeated fields */ + alltypes.rep_int32.funcs.decode = &read_repeated_varint; + alltypes.rep_int32.arg = rep_int32; + + alltypes.rep_int64.funcs.decode = &read_repeated_varint; + alltypes.rep_int64.arg = rep_int64; + + alltypes.rep_uint32.funcs.decode = &read_repeated_varint; + alltypes.rep_uint32.arg = rep_uint32; + + alltypes.rep_uint64.funcs.decode = &read_repeated_varint; + alltypes.rep_uint64.arg = rep_uint64; + + alltypes.rep_sint32.funcs.decode = &read_repeated_svarint; + alltypes.rep_sint32.arg = rep_sint32; + + alltypes.rep_sint64.funcs.decode = &read_repeated_svarint; + alltypes.rep_sint64.arg = rep_sint64; + + alltypes.rep_bool.funcs.decode = &read_repeated_varint; + alltypes.rep_bool.arg = rep_bool; + + alltypes.rep_fixed32.funcs.decode = &read_repeated_fixed32; + alltypes.rep_fixed32.arg = rep_fixed32; + + alltypes.rep_sfixed32.funcs.decode = &read_repeated_fixed32; + alltypes.rep_sfixed32.arg = rep_sfixed32; + + alltypes.rep_float.funcs.decode = &read_repeated_fixed32; + alltypes.rep_float.arg = rep_float; + + alltypes.rep_fixed64.funcs.decode = &read_repeated_fixed64; + alltypes.rep_fixed64.arg = rep_fixed64; + + alltypes.rep_sfixed64.funcs.decode = &read_repeated_fixed64; + alltypes.rep_sfixed64.arg = rep_sfixed64; + + alltypes.rep_double.funcs.decode = &read_repeated_fixed64; + alltypes.rep_double.arg = rep_double; + + alltypes.rep_string.funcs.decode = &read_repeated_string; + alltypes.rep_string.arg = rep_string; + + alltypes.rep_bytes.funcs.decode = &read_repeated_string; + alltypes.rep_bytes.arg = rep_bytes; + + alltypes.rep_submsg.funcs.decode = &read_repeated_submsg; + alltypes.rep_submsg.arg = rep_submsg; + + alltypes.rep_enum.funcs.decode = &read_repeated_varint; + alltypes.rep_enum.arg = rep_enum; + + alltypes.rep_emptymsg.funcs.decode = &read_emptymsg; + + alltypes.req_limits.funcs.decode = &read_limits; + + alltypes.end.funcs.decode = &read_varint; + alltypes.end.arg = (void*)1099; + + /* Bind callbacks for optional fields */ + if (mode == 1) + { + alltypes.sng_int32.funcs.decode = &read_varint; + alltypes.sng_int32.arg = (void*)3041; + + alltypes.sng_int64.funcs.decode = &read_varint; + alltypes.sng_int64.arg = (void*)3042; + + alltypes.sng_uint32.funcs.decode = &read_varint; + alltypes.sng_uint32.arg = (void*)3043; + + alltypes.sng_uint64.funcs.decode = &read_varint; + alltypes.sng_uint64.arg = (void*)3044; + + alltypes.sng_sint32.funcs.decode = &read_svarint; + alltypes.sng_sint32.arg = (void*)3045; + + alltypes.sng_sint64.funcs.decode = &read_svarint; + alltypes.sng_sint64.arg = (void*)3046; + + alltypes.sng_bool.funcs.decode = &read_varint; + alltypes.sng_bool.arg = (void*)true; + + alltypes.sng_fixed32.funcs.decode = &read_fixed32; + alltypes.sng_fixed32.arg = &sng_fixed32; + + alltypes.sng_sfixed32.funcs.decode = &read_fixed32; + alltypes.sng_sfixed32.arg = &sng_sfixed32; + + alltypes.sng_float.funcs.decode = &read_fixed32; + alltypes.sng_float.arg = &sng_float; + + alltypes.sng_fixed64.funcs.decode = &read_fixed64; + alltypes.sng_fixed64.arg = &sng_fixed64; + + alltypes.sng_sfixed64.funcs.decode = &read_fixed64; + alltypes.sng_sfixed64.arg = &sng_sfixed64; + + alltypes.sng_double.funcs.decode = &read_fixed64; + alltypes.sng_double.arg = &sng_double; + + alltypes.sng_string.funcs.decode = &read_string; + alltypes.sng_string.arg = "3054"; + + alltypes.sng_bytes.funcs.decode = &read_string; + alltypes.sng_bytes.arg = "3055"; + + alltypes.sng_submsg.funcs.decode = &read_submsg; + alltypes.sng_submsg.arg = &sng_submsg; + + alltypes.sng_enum.funcs.decode = &read_varint; + alltypes.sng_enum.arg = (void*)MyEnum_Truth; + + alltypes.sng_emptymsg.funcs.decode = &read_emptymsg; + + alltypes.oneof_msg1.funcs.decode = &read_submsg; + alltypes.oneof_msg1.arg = &oneof_msg1; + } + + status = pb_decode(stream, AllTypes_fields, &alltypes); + +#ifdef PB_ENABLE_MALLOC + /* Just to check for any interference between pb_release() and callback fields */ + pb_release(AllTypes_fields, &alltypes); +#endif + + return status; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[1024]; + size_t count; + pb_istream_t stream; + + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/alltypes_proto3_callback/encode_alltypes_callback.c b/tests/alltypes_proto3_callback/encode_alltypes_callback.c new file mode 100644 index 0000000..8c7bdd6 --- /dev/null +++ b/tests/alltypes_proto3_callback/encode_alltypes_callback.c @@ -0,0 +1,343 @@ +/* Attempts to test all the datatypes supported by ProtoBuf when used as callback fields. + * Note that normally there would be no reason to use callback fields for this, + * because each encoder defined here only gives a single field. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pb_encode.h> +#include "alltypes.pb.h" +#include "test_helpers.h" + +static bool write_varint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, (long)*arg); +} + +static bool write_svarint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, (long)*arg); +} + +static bool write_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_fixed32(stream, *arg); +} + +static bool write_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_fixed64(stream, *arg); +} + +static bool write_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, *arg, strlen(*arg)); +} + +static bool write_submsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, *arg); +} + +static bool write_emptymsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + EmptyMessage emptymsg = {0}; + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg); +} + +static bool write_repeated_varint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_varint(stream, (long)*arg); +} + +static bool write_repeated_svarint(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_svarint(stream, (long)*arg); +} + +static bool write_repeated_fixed32(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + uint32_t dummy = 0; + + /* Make it a packed field */ + return pb_encode_tag(stream, PB_WT_STRING, field->tag) && + pb_encode_varint(stream, 5 * 4) && /* Number of bytes */ + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, &dummy) && + pb_encode_fixed32(stream, *arg); +} + +static bool write_repeated_fixed64(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + uint64_t dummy = 0; + + /* Make it a packed field */ + return pb_encode_tag(stream, PB_WT_STRING, field->tag) && + pb_encode_varint(stream, 5 * 8) && /* Number of bytes */ + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, &dummy) && + pb_encode_fixed64(stream, *arg); +} + +static bool write_repeated_string(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + return pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, 0, 0) && + pb_encode_tag_for_field(stream, field) && + pb_encode_string(stream, *arg, strlen(*arg)); +} + +static bool write_repeated_submsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + SubMessage dummy = {""}; + + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, &dummy) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, SubMessage_fields, *arg); +} + +static bool write_limits(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + Limits limits = {0}; + limits.int32_min = INT32_MIN; + limits.int32_max = INT32_MAX; + limits.uint32_min = 0; + limits.uint32_max = UINT32_MAX; + limits.int64_min = INT64_MIN; + limits.int64_max = INT64_MAX; + limits.uint64_min = 0; + limits.uint64_max = UINT64_MAX; + limits.enum_min = HugeEnum_Negative; + limits.enum_max = HugeEnum_Positive; + + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, Limits_fields, &limits); +} + +static bool write_repeated_emptymsg(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + EmptyMessage emptymsg = {0}; + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg) && + pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, EmptyMessage_fields, &emptymsg); +} + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Values for use from callbacks through pointers. */ + uint32_t rep_fixed32 = 2008; + int32_t rep_sfixed32 = -2009; + float rep_float = 2010.0f; + uint64_t rep_fixed64 = 2011; + int64_t rep_sfixed64 = -2012; + double rep_double = 2013.0; + SubMessage rep_submsg = {"2016", 2016, 2016}; + + uint32_t sng_fixed32 = 3048; + int32_t sng_sfixed32 = 3049; + float sng_float = 3050.0f; + uint64_t sng_fixed64 = 3051; + int64_t sng_sfixed64 = 3052; + double sng_double = 3053.0f; + SubMessage sng_submsg = {"3056", 3056}; + + SubMessage oneof_msg1 = {"4059", 4059}; + + AllTypes alltypes = AllTypes_init_zero; + + /* Bind callbacks for repeated fields */ + alltypes.rep_int32.funcs.encode = &write_repeated_varint; + alltypes.rep_int32.arg = (void*)-2001; + + alltypes.rep_int64.funcs.encode = &write_repeated_varint; + alltypes.rep_int64.arg = (void*)-2002; + + alltypes.rep_uint32.funcs.encode = &write_repeated_varint; + alltypes.rep_uint32.arg = (void*)2003; + + alltypes.rep_uint64.funcs.encode = &write_repeated_varint; + alltypes.rep_uint64.arg = (void*)2004; + + alltypes.rep_sint32.funcs.encode = &write_repeated_svarint; + alltypes.rep_sint32.arg = (void*)-2005; + + alltypes.rep_sint64.funcs.encode = &write_repeated_svarint; + alltypes.rep_sint64.arg = (void*)-2006; + + alltypes.rep_bool.funcs.encode = &write_repeated_varint; + alltypes.rep_bool.arg = (void*)true; + + alltypes.rep_fixed32.funcs.encode = &write_repeated_fixed32; + alltypes.rep_fixed32.arg = &rep_fixed32; + + alltypes.rep_sfixed32.funcs.encode = &write_repeated_fixed32; + alltypes.rep_sfixed32.arg = &rep_sfixed32; + + alltypes.rep_float.funcs.encode = &write_repeated_fixed32; + alltypes.rep_float.arg = &rep_float; + + alltypes.rep_fixed64.funcs.encode = &write_repeated_fixed64; + alltypes.rep_fixed64.arg = &rep_fixed64; + + alltypes.rep_sfixed64.funcs.encode = &write_repeated_fixed64; + alltypes.rep_sfixed64.arg = &rep_sfixed64; + + alltypes.rep_double.funcs.encode = &write_repeated_fixed64; + alltypes.rep_double.arg = &rep_double; + + alltypes.rep_string.funcs.encode = &write_repeated_string; + alltypes.rep_string.arg = "2014"; + + alltypes.rep_bytes.funcs.encode = &write_repeated_string; + alltypes.rep_bytes.arg = "2015"; + + alltypes.rep_submsg.funcs.encode = &write_repeated_submsg; + alltypes.rep_submsg.arg = &rep_submsg; + + alltypes.rep_enum.funcs.encode = &write_repeated_varint; + alltypes.rep_enum.arg = (void*)MyEnum_Truth; + + alltypes.rep_emptymsg.funcs.encode = &write_repeated_emptymsg; + + alltypes.rep_fbytes.funcs.encode = &write_repeated_string; + alltypes.rep_fbytes.arg = "2019"; + + alltypes.req_limits.funcs.encode = &write_limits; + + /* Bind callbacks for singular fields */ + if (mode != 0) + { + alltypes.sng_int32.funcs.encode = &write_varint; + alltypes.sng_int32.arg = (void*)3041; + + alltypes.sng_int64.funcs.encode = &write_varint; + alltypes.sng_int64.arg = (void*)3042; + + alltypes.sng_uint32.funcs.encode = &write_varint; + alltypes.sng_uint32.arg = (void*)3043; + + alltypes.sng_uint64.funcs.encode = &write_varint; + alltypes.sng_uint64.arg = (void*)3044; + + alltypes.sng_sint32.funcs.encode = &write_svarint; + alltypes.sng_sint32.arg = (void*)3045; + + alltypes.sng_sint64.funcs.encode = &write_svarint; + alltypes.sng_sint64.arg = (void*)3046; + + alltypes.sng_bool.funcs.encode = &write_varint; + alltypes.sng_bool.arg = (void*)true; + + alltypes.sng_fixed32.funcs.encode = &write_fixed32; + alltypes.sng_fixed32.arg = &sng_fixed32; + + alltypes.sng_sfixed32.funcs.encode = &write_fixed32; + alltypes.sng_sfixed32.arg = &sng_sfixed32; + + alltypes.sng_float.funcs.encode = &write_fixed32; + alltypes.sng_float.arg = &sng_float; + + alltypes.sng_fixed64.funcs.encode = &write_fixed64; + alltypes.sng_fixed64.arg = &sng_fixed64; + + alltypes.sng_sfixed64.funcs.encode = &write_fixed64; + alltypes.sng_sfixed64.arg = &sng_sfixed64; + + alltypes.sng_double.funcs.encode = &write_fixed64; + alltypes.sng_double.arg = &sng_double; + + alltypes.sng_string.funcs.encode = &write_string; + alltypes.sng_string.arg = "3054"; + + alltypes.sng_bytes.funcs.encode = &write_string; + alltypes.sng_bytes.arg = "3055"; + + alltypes.sng_submsg.funcs.encode = &write_submsg; + alltypes.sng_submsg.arg = &sng_submsg; + + alltypes.sng_enum.funcs.encode = &write_varint; + alltypes.sng_enum.arg = (void*)MyEnum_Truth; + + alltypes.sng_emptymsg.funcs.encode = &write_emptymsg; + + alltypes.sng_fbytes.funcs.encode = &write_string; + alltypes.sng_fbytes.arg = "3059"; + + alltypes.oneof_msg1.funcs.encode = &write_submsg; + alltypes.oneof_msg1.arg = &oneof_msg1; + } + + alltypes.end.funcs.encode = &write_varint; + alltypes.end.arg = (void*)1099; + + { + uint8_t buffer[2048]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } + } +} diff --git a/tests/anonymous_oneof/SConscript b/tests/anonymous_oneof/SConscript new file mode 100644 index 0000000..20fd1cc --- /dev/null +++ b/tests/anonymous_oneof/SConscript @@ -0,0 +1,30 @@ +# Test anonymous_oneof generator option + +Import('env') + +import re + +match = None +if 'PROTOC_VERSION' in env: + match = re.search('([0-9]+).([0-9]+).([0-9]+)', env['PROTOC_VERSION']) + +if match: + version = list(map(int, match.groups())) + +# Oneof is supported by protoc >= 2.6.0 +if env.GetOption('clean') or (match and (version[0] > 2 or (version[0] == 2 and version[1] >= 6))): + # Anonymous oneofs are supported by clang and gcc + if 'clang' in env['CC'] or 'gcc' in env['CC']: + env2 = env.Clone() + if '-pedantic' in env2['CFLAGS']: + env2['CFLAGS'].remove('-pedantic') + env2.NanopbProto('oneof') + + dec = env2.Program(['decode_oneof.c', + 'oneof.pb.c', + '$COMMON/pb_decode.o', + '$COMMON/pb_common.o']) + + env2.RunTest("message1.txt", [dec, '$BUILD/oneof/message1.pb'], ARGS = ['1']) + env2.RunTest("message2.txt", [dec, '$BUILD/oneof/message2.pb'], ARGS = ['2']) + env2.RunTest("message3.txt", [dec, '$BUILD/oneof/message3.pb'], ARGS = ['3']) diff --git a/tests/anonymous_oneof/decode_oneof.c b/tests/anonymous_oneof/decode_oneof.c new file mode 100644 index 0000000..0f774db --- /dev/null +++ b/tests/anonymous_oneof/decode_oneof.c @@ -0,0 +1,88 @@ +/* Decode a message using oneof fields */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pb_decode.h> +#include "oneof.pb.h" +#include "test_helpers.h" +#include "unittests.h" + +/* Test the 'AnonymousOneOfMessage' */ +int test_oneof_1(pb_istream_t *stream, int option) +{ + AnonymousOneOfMessage msg; + int status = 0; + + /* To better catch initialization errors */ + memset(&msg, 0xAA, sizeof(msg)); + + if (!pb_decode(stream, AnonymousOneOfMessage_fields, &msg)) + { + printf("Decoding failed: %s\n", PB_GET_ERROR(stream)); + return 1; + } + + /* Check that the basic fields work normally */ + TEST(msg.prefix == 123); + TEST(msg.suffix == 321); + + /* Check that we got the right oneof according to command line */ + if (option == 1) + { + TEST(msg.which_values == AnonymousOneOfMessage_first_tag); + TEST(msg.first == 999); + } + else if (option == 2) + { + TEST(msg.which_values == AnonymousOneOfMessage_second_tag); + TEST(strcmp(msg.second, "abcd") == 0); + } + else if (option == 3) + { + TEST(msg.which_values == AnonymousOneOfMessage_third_tag); + TEST(msg.third.array[0] == 1); + TEST(msg.third.array[1] == 2); + TEST(msg.third.array[2] == 3); + TEST(msg.third.array[3] == 4); + TEST(msg.third.array[4] == 5); + } + + return status; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[AnonymousOneOfMessage_size]; + size_t count; + int option; + + if (argc != 2) + { + fprintf(stderr, "Usage: decode_oneof [number]\n"); + return 1; + } + option = atoi(argv[1]); + + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + if (!feof(stdin)) + { + printf("Message does not fit in buffer\n"); + return 1; + } + + { + int status = 0; + pb_istream_t stream; + + stream = pb_istream_from_buffer(buffer, count); + status = test_oneof_1(&stream, option); + + if (status != 0) + return status; + } + + return 0; +} diff --git a/tests/anonymous_oneof/oneof.proto b/tests/anonymous_oneof/oneof.proto new file mode 100644 index 0000000..d56285c --- /dev/null +++ b/tests/anonymous_oneof/oneof.proto @@ -0,0 +1,23 @@ +syntax = "proto2"; + +import 'nanopb.proto'; + +message SubMessage +{ + repeated int32 array = 1 [(nanopb).max_count = 8]; +} + +/* Oneof in a message with other fields */ +message AnonymousOneOfMessage +{ + option (nanopb_msgopt).anonymous_oneof = true; + required int32 prefix = 1; + oneof values + { + int32 first = 5; + string second = 6 [(nanopb).max_size = 8]; + SubMessage third = 7; + } + required int32 suffix = 99; +} + diff --git a/tests/backwards_compatibility/SConscript b/tests/backwards_compatibility/SConscript index 777ef40..81b0318 100644 --- a/tests/backwards_compatibility/SConscript +++ b/tests/backwards_compatibility/SConscript @@ -3,8 +3,8 @@ Import("env") -enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_legacy.c", "alltypes_legacy.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(enc) env.RunTest([dec, "encode_legacy.output"]) diff --git a/tests/backwards_compatibility/alltypes_legacy.c b/tests/backwards_compatibility/alltypes_legacy.c index 9134d5e..7311fd4 100644 --- a/tests/backwards_compatibility/alltypes_legacy.c +++ b/tests/backwards_compatibility/alltypes_legacy.c @@ -1,27 +1,37 @@ /* Automatically generated nanopb constant definitions */ -/* Generated by 0.2.0-dev at Sun Feb 17 00:09:53 2013. */ -/* This is a file generated using nanopb-0.2.0-dev. - * It is used as a part of test suite in order to detect any - * incompatible changes made to the generator in future versions. - */ +/* Generated by nanopb-0.3.0-dev at Tue Aug 19 17:53:24 2014. */ #include "alltypes_legacy.h" +#if PB_PROTO_HEADER_VERSION != 30 +#error Regenerate this file with the current version of nanopb generator. +#endif + const char SubMessage_substuff1_default[16] = "1"; const int32_t SubMessage_substuff2_default = 2; -const uint32_t SubMessage_substuff3_default = 3; +const uint32_t SubMessage_substuff3_default = 3u; +const int32_t Limits_int32_min_default = 2147483647; +const int32_t Limits_int32_max_default = -2147483647; +const uint32_t Limits_uint32_min_default = 4294967295u; +const uint32_t Limits_uint32_max_default = 0u; +const int64_t Limits_int64_min_default = 9223372036854775807ll; +const int64_t Limits_int64_max_default = -9223372036854775807ll; +const uint64_t Limits_uint64_min_default = 18446744073709551615ull; +const uint64_t Limits_uint64_max_default = 0ull; +const HugeEnum Limits_enum_min_default = HugeEnum_Positive; +const HugeEnum Limits_enum_max_default = HugeEnum_Negative; const int32_t AllTypes_opt_int32_default = 4041; -const int64_t AllTypes_opt_int64_default = 4042; -const uint32_t AllTypes_opt_uint32_default = 4043; -const uint64_t AllTypes_opt_uint64_default = 4044; +const int64_t AllTypes_opt_int64_default = 4042ll; +const uint32_t AllTypes_opt_uint32_default = 4043u; +const uint64_t AllTypes_opt_uint64_default = 4044ull; const int32_t AllTypes_opt_sint32_default = 4045; const int64_t AllTypes_opt_sint64_default = 4046; const bool AllTypes_opt_bool_default = false; -const uint32_t AllTypes_opt_fixed32_default = 4048; +const uint32_t AllTypes_opt_fixed32_default = 4048u; const int32_t AllTypes_opt_sfixed32_default = 4049; const float AllTypes_opt_float_default = 4050; -const uint64_t AllTypes_opt_fixed64_default = 4051; -const int64_t AllTypes_opt_sfixed64_default = 4052; +const uint64_t AllTypes_opt_fixed64_default = 4051ull; +const int64_t AllTypes_opt_sfixed64_default = 4052ll; const double AllTypes_opt_double_default = 4053; const char AllTypes_opt_string_default[16] = "4054"; const AllTypes_opt_bytes_t AllTypes_opt_bytes_default = {4, {0x34,0x30,0x35,0x35}}; @@ -29,65 +39,115 @@ const MyEnum AllTypes_opt_enum_default = MyEnum_Second; const pb_field_t SubMessage_fields[4] = { - PB_FIELD( 1, STRING , REQUIRED, STATIC, SubMessage, substuff1, substuff1, &SubMessage_substuff1_default), - PB_FIELD( 2, INT32 , REQUIRED, STATIC, SubMessage, substuff2, substuff1, &SubMessage_substuff2_default), - PB_FIELD( 3, FIXED32 , OPTIONAL, STATIC, SubMessage, substuff3, substuff2, &SubMessage_substuff3_default), + PB_FIELD( 1, STRING , REQUIRED, STATIC , FIRST, SubMessage, substuff1, substuff1, &SubMessage_substuff1_default), + PB_FIELD( 2, INT32 , REQUIRED, STATIC , OTHER, SubMessage, substuff2, substuff1, &SubMessage_substuff2_default), + PB_FIELD( 3, FIXED32 , OPTIONAL, STATIC , OTHER, SubMessage, substuff3, substuff2, &SubMessage_substuff3_default), + PB_LAST_FIELD +}; + +const pb_field_t EmptyMessage_fields[1] = { + PB_LAST_FIELD +}; + +const pb_field_t Limits_fields[11] = { + PB_FIELD( 1, INT32 , REQUIRED, STATIC , FIRST, Limits, int32_min, int32_min, &Limits_int32_min_default), + PB_FIELD( 2, INT32 , REQUIRED, STATIC , OTHER, Limits, int32_max, int32_min, &Limits_int32_max_default), + PB_FIELD( 3, UINT32 , REQUIRED, STATIC , OTHER, Limits, uint32_min, int32_max, &Limits_uint32_min_default), + PB_FIELD( 4, UINT32 , REQUIRED, STATIC , OTHER, Limits, uint32_max, uint32_min, &Limits_uint32_max_default), + PB_FIELD( 5, INT64 , REQUIRED, STATIC , OTHER, Limits, int64_min, uint32_max, &Limits_int64_min_default), + PB_FIELD( 6, INT64 , REQUIRED, STATIC , OTHER, Limits, int64_max, int64_min, &Limits_int64_max_default), + PB_FIELD( 7, UINT64 , REQUIRED, STATIC , OTHER, Limits, uint64_min, int64_max, &Limits_uint64_min_default), + PB_FIELD( 8, UINT64 , REQUIRED, STATIC , OTHER, Limits, uint64_max, uint64_min, &Limits_uint64_max_default), + PB_FIELD( 9, ENUM , REQUIRED, STATIC , OTHER, Limits, enum_min, uint64_max, &Limits_enum_min_default), + PB_FIELD( 10, ENUM , REQUIRED, STATIC , OTHER, Limits, enum_max, enum_min, &Limits_enum_max_default), PB_LAST_FIELD }; -const pb_field_t AllTypes_fields[53] = { - PB_FIELD( 1, INT32 , REQUIRED, STATIC, AllTypes, req_int32, req_int32, 0), - PB_FIELD( 2, INT64 , REQUIRED, STATIC, AllTypes, req_int64, req_int32, 0), - PB_FIELD( 3, UINT32 , REQUIRED, STATIC, AllTypes, req_uint32, req_int64, 0), - PB_FIELD( 4, UINT64 , REQUIRED, STATIC, AllTypes, req_uint64, req_uint32, 0), - PB_FIELD( 5, SINT32 , REQUIRED, STATIC, AllTypes, req_sint32, req_uint64, 0), - PB_FIELD( 6, SINT64 , REQUIRED, STATIC, AllTypes, req_sint64, req_sint32, 0), - PB_FIELD( 7, BOOL , REQUIRED, STATIC, AllTypes, req_bool, req_sint64, 0), - PB_FIELD( 8, FIXED32 , REQUIRED, STATIC, AllTypes, req_fixed32, req_bool, 0), - PB_FIELD( 9, SFIXED32, REQUIRED, STATIC, AllTypes, req_sfixed32, req_fixed32, 0), - PB_FIELD( 10, FLOAT , REQUIRED, STATIC, AllTypes, req_float, req_sfixed32, 0), - PB_FIELD( 11, FIXED64 , REQUIRED, STATIC, AllTypes, req_fixed64, req_float, 0), - PB_FIELD( 12, SFIXED64, REQUIRED, STATIC, AllTypes, req_sfixed64, req_fixed64, 0), - PB_FIELD( 13, DOUBLE , REQUIRED, STATIC, AllTypes, req_double, req_sfixed64, 0), - PB_FIELD( 14, STRING , REQUIRED, STATIC, AllTypes, req_string, req_double, 0), - PB_FIELD( 15, BYTES , REQUIRED, STATIC, AllTypes, req_bytes, req_string, 0), - PB_FIELD( 16, MESSAGE , REQUIRED, STATIC, AllTypes, req_submsg, req_bytes, &SubMessage_fields), - PB_FIELD( 17, ENUM , REQUIRED, STATIC, AllTypes, req_enum, req_submsg, 0), - PB_FIELD( 21, INT32 , REPEATED, STATIC, AllTypes, rep_int32, req_enum, 0), - PB_FIELD( 22, INT64 , REPEATED, STATIC, AllTypes, rep_int64, rep_int32, 0), - PB_FIELD( 23, UINT32 , REPEATED, STATIC, AllTypes, rep_uint32, rep_int64, 0), - PB_FIELD( 24, UINT64 , REPEATED, STATIC, AllTypes, rep_uint64, rep_uint32, 0), - PB_FIELD( 25, SINT32 , REPEATED, STATIC, AllTypes, rep_sint32, rep_uint64, 0), - PB_FIELD( 26, SINT64 , REPEATED, STATIC, AllTypes, rep_sint64, rep_sint32, 0), - PB_FIELD( 27, BOOL , REPEATED, STATIC, AllTypes, rep_bool, rep_sint64, 0), - PB_FIELD( 28, FIXED32 , REPEATED, STATIC, AllTypes, rep_fixed32, rep_bool, 0), - PB_FIELD( 29, SFIXED32, REPEATED, STATIC, AllTypes, rep_sfixed32, rep_fixed32, 0), - PB_FIELD( 30, FLOAT , REPEATED, STATIC, AllTypes, rep_float, rep_sfixed32, 0), - PB_FIELD( 31, FIXED64 , REPEATED, STATIC, AllTypes, rep_fixed64, rep_float, 0), - PB_FIELD( 32, SFIXED64, REPEATED, STATIC, AllTypes, rep_sfixed64, rep_fixed64, 0), - PB_FIELD( 33, DOUBLE , REPEATED, STATIC, AllTypes, rep_double, rep_sfixed64, 0), - PB_FIELD( 34, STRING , REPEATED, STATIC, AllTypes, rep_string, rep_double, 0), - PB_FIELD( 35, BYTES , REPEATED, STATIC, AllTypes, rep_bytes, rep_string, 0), - PB_FIELD( 36, MESSAGE , REPEATED, STATIC, AllTypes, rep_submsg, rep_bytes, &SubMessage_fields), - PB_FIELD( 37, ENUM , REPEATED, STATIC, AllTypes, rep_enum, rep_submsg, 0), - PB_FIELD( 41, INT32 , OPTIONAL, STATIC, AllTypes, opt_int32, rep_enum, &AllTypes_opt_int32_default), - PB_FIELD( 42, INT64 , OPTIONAL, STATIC, AllTypes, opt_int64, opt_int32, &AllTypes_opt_int64_default), - PB_FIELD( 43, UINT32 , OPTIONAL, STATIC, AllTypes, opt_uint32, opt_int64, &AllTypes_opt_uint32_default), - PB_FIELD( 44, UINT64 , OPTIONAL, STATIC, AllTypes, opt_uint64, opt_uint32, &AllTypes_opt_uint64_default), - PB_FIELD( 45, SINT32 , OPTIONAL, STATIC, AllTypes, opt_sint32, opt_uint64, &AllTypes_opt_sint32_default), - PB_FIELD( 46, SINT64 , OPTIONAL, STATIC, AllTypes, opt_sint64, opt_sint32, &AllTypes_opt_sint64_default), - PB_FIELD( 47, BOOL , OPTIONAL, STATIC, AllTypes, opt_bool, opt_sint64, &AllTypes_opt_bool_default), - PB_FIELD( 48, FIXED32 , OPTIONAL, STATIC, AllTypes, opt_fixed32, opt_bool, &AllTypes_opt_fixed32_default), - PB_FIELD( 49, SFIXED32, OPTIONAL, STATIC, AllTypes, opt_sfixed32, opt_fixed32, &AllTypes_opt_sfixed32_default), - PB_FIELD( 50, FLOAT , OPTIONAL, STATIC, AllTypes, opt_float, opt_sfixed32, &AllTypes_opt_float_default), - PB_FIELD( 51, FIXED64 , OPTIONAL, STATIC, AllTypes, opt_fixed64, opt_float, &AllTypes_opt_fixed64_default), - PB_FIELD( 52, SFIXED64, OPTIONAL, STATIC, AllTypes, opt_sfixed64, opt_fixed64, &AllTypes_opt_sfixed64_default), - PB_FIELD( 53, DOUBLE , OPTIONAL, STATIC, AllTypes, opt_double, opt_sfixed64, &AllTypes_opt_double_default), - PB_FIELD( 54, STRING , OPTIONAL, STATIC, AllTypes, opt_string, opt_double, &AllTypes_opt_string_default), - PB_FIELD( 55, BYTES , OPTIONAL, STATIC, AllTypes, opt_bytes, opt_string, &AllTypes_opt_bytes_default), - PB_FIELD( 56, MESSAGE , OPTIONAL, STATIC, AllTypes, opt_submsg, opt_bytes, &SubMessage_fields), - PB_FIELD( 57, ENUM , OPTIONAL, STATIC, AllTypes, opt_enum, opt_submsg, &AllTypes_opt_enum_default), - PB_FIELD( 99, INT32 , REQUIRED, STATIC, AllTypes, end, opt_enum, 0), +const pb_field_t AllTypes_fields[54] = { + PB_FIELD( 1, INT32 , REQUIRED, STATIC , FIRST, AllTypes, req_int32, req_int32, 0), + PB_FIELD( 2, INT64 , REQUIRED, STATIC , OTHER, AllTypes, req_int64, req_int32, 0), + PB_FIELD( 3, UINT32 , REQUIRED, STATIC , OTHER, AllTypes, req_uint32, req_int64, 0), + PB_FIELD( 4, UINT64 , REQUIRED, STATIC , OTHER, AllTypes, req_uint64, req_uint32, 0), + PB_FIELD( 5, SINT32 , REQUIRED, STATIC , OTHER, AllTypes, req_sint32, req_uint64, 0), + PB_FIELD( 6, SINT64 , REQUIRED, STATIC , OTHER, AllTypes, req_sint64, req_sint32, 0), + PB_FIELD( 7, BOOL , REQUIRED, STATIC , OTHER, AllTypes, req_bool, req_sint64, 0), + PB_FIELD( 8, FIXED32 , REQUIRED, STATIC , OTHER, AllTypes, req_fixed32, req_bool, 0), + PB_FIELD( 9, SFIXED32, REQUIRED, STATIC , OTHER, AllTypes, req_sfixed32, req_fixed32, 0), + PB_FIELD( 10, FLOAT , REQUIRED, STATIC , OTHER, AllTypes, req_float, req_sfixed32, 0), + PB_FIELD( 11, FIXED64 , REQUIRED, STATIC , OTHER, AllTypes, req_fixed64, req_float, 0), + PB_FIELD( 12, SFIXED64, REQUIRED, STATIC , OTHER, AllTypes, req_sfixed64, req_fixed64, 0), + PB_FIELD( 13, DOUBLE , REQUIRED, STATIC , OTHER, AllTypes, req_double, req_sfixed64, 0), + PB_FIELD( 14, STRING , REQUIRED, STATIC , OTHER, AllTypes, req_string, req_double, 0), + PB_FIELD( 15, BYTES , REQUIRED, STATIC , OTHER, AllTypes, req_bytes, req_string, 0), + PB_FIELD( 16, MESSAGE , REQUIRED, STATIC , OTHER, AllTypes, req_submsg, req_bytes, &SubMessage_fields), + PB_FIELD( 17, ENUM , REQUIRED, STATIC , OTHER, AllTypes, req_enum, req_submsg, 0), + PB_FIELD( 21, INT32 , REPEATED, STATIC , OTHER, AllTypes, rep_int32, req_enum, 0), + PB_FIELD( 22, INT64 , REPEATED, STATIC , OTHER, AllTypes, rep_int64, rep_int32, 0), + PB_FIELD( 23, UINT32 , REPEATED, STATIC , OTHER, AllTypes, rep_uint32, rep_int64, 0), + PB_FIELD( 24, UINT64 , REPEATED, STATIC , OTHER, AllTypes, rep_uint64, rep_uint32, 0), + PB_FIELD( 25, SINT32 , REPEATED, STATIC , OTHER, AllTypes, rep_sint32, rep_uint64, 0), + PB_FIELD( 26, SINT64 , REPEATED, STATIC , OTHER, AllTypes, rep_sint64, rep_sint32, 0), + PB_FIELD( 27, BOOL , REPEATED, STATIC , OTHER, AllTypes, rep_bool, rep_sint64, 0), + PB_FIELD( 28, FIXED32 , REPEATED, STATIC , OTHER, AllTypes, rep_fixed32, rep_bool, 0), + PB_FIELD( 29, SFIXED32, REPEATED, STATIC , OTHER, AllTypes, rep_sfixed32, rep_fixed32, 0), + PB_FIELD( 30, FLOAT , REPEATED, STATIC , OTHER, AllTypes, rep_float, rep_sfixed32, 0), + PB_FIELD( 31, FIXED64 , REPEATED, STATIC , OTHER, AllTypes, rep_fixed64, rep_float, 0), + PB_FIELD( 32, SFIXED64, REPEATED, STATIC , OTHER, AllTypes, rep_sfixed64, rep_fixed64, 0), + PB_FIELD( 33, DOUBLE , REPEATED, STATIC , OTHER, AllTypes, rep_double, rep_sfixed64, 0), + PB_FIELD( 34, STRING , REPEATED, STATIC , OTHER, AllTypes, rep_string, rep_double, 0), + PB_FIELD( 35, BYTES , REPEATED, STATIC , OTHER, AllTypes, rep_bytes, rep_string, 0), + PB_FIELD( 36, MESSAGE , REPEATED, STATIC , OTHER, AllTypes, rep_submsg, rep_bytes, &SubMessage_fields), + PB_FIELD( 37, ENUM , REPEATED, STATIC , OTHER, AllTypes, rep_enum, rep_submsg, 0), + PB_FIELD( 41, INT32 , OPTIONAL, STATIC , OTHER, AllTypes, opt_int32, rep_enum, &AllTypes_opt_int32_default), + PB_FIELD( 42, INT64 , OPTIONAL, STATIC , OTHER, AllTypes, opt_int64, opt_int32, &AllTypes_opt_int64_default), + PB_FIELD( 43, UINT32 , OPTIONAL, STATIC , OTHER, AllTypes, opt_uint32, opt_int64, &AllTypes_opt_uint32_default), + PB_FIELD( 44, UINT64 , OPTIONAL, STATIC , OTHER, AllTypes, opt_uint64, opt_uint32, &AllTypes_opt_uint64_default), + PB_FIELD( 45, SINT32 , OPTIONAL, STATIC , OTHER, AllTypes, opt_sint32, opt_uint64, &AllTypes_opt_sint32_default), + PB_FIELD( 46, SINT64 , OPTIONAL, STATIC , OTHER, AllTypes, opt_sint64, opt_sint32, &AllTypes_opt_sint64_default), + PB_FIELD( 47, BOOL , OPTIONAL, STATIC , OTHER, AllTypes, opt_bool, opt_sint64, &AllTypes_opt_bool_default), + PB_FIELD( 48, FIXED32 , OPTIONAL, STATIC , OTHER, AllTypes, opt_fixed32, opt_bool, &AllTypes_opt_fixed32_default), + PB_FIELD( 49, SFIXED32, OPTIONAL, STATIC , OTHER, AllTypes, opt_sfixed32, opt_fixed32, &AllTypes_opt_sfixed32_default), + PB_FIELD( 50, FLOAT , OPTIONAL, STATIC , OTHER, AllTypes, opt_float, opt_sfixed32, &AllTypes_opt_float_default), + PB_FIELD( 51, FIXED64 , OPTIONAL, STATIC , OTHER, AllTypes, opt_fixed64, opt_float, &AllTypes_opt_fixed64_default), + PB_FIELD( 52, SFIXED64, OPTIONAL, STATIC , OTHER, AllTypes, opt_sfixed64, opt_fixed64, &AllTypes_opt_sfixed64_default), + PB_FIELD( 53, DOUBLE , OPTIONAL, STATIC , OTHER, AllTypes, opt_double, opt_sfixed64, &AllTypes_opt_double_default), + PB_FIELD( 54, STRING , OPTIONAL, STATIC , OTHER, AllTypes, opt_string, opt_double, &AllTypes_opt_string_default), + PB_FIELD( 55, BYTES , OPTIONAL, STATIC , OTHER, AllTypes, opt_bytes, opt_string, &AllTypes_opt_bytes_default), + PB_FIELD( 56, MESSAGE , OPTIONAL, STATIC , OTHER, AllTypes, opt_submsg, opt_bytes, &SubMessage_fields), + PB_FIELD( 57, ENUM , OPTIONAL, STATIC , OTHER, AllTypes, opt_enum, opt_submsg, &AllTypes_opt_enum_default), + PB_FIELD( 99, INT32 , REQUIRED, STATIC , OTHER, AllTypes, end, opt_enum, 0), + PB_FIELD(200, EXTENSION, OPTIONAL, CALLBACK, OTHER, AllTypes, extensions, end, 0), PB_LAST_FIELD }; + +/* Check that field information fits in pb_field_t */ +#if !defined(PB_FIELD_32BIT) +/* If you get an error here, it means that you need to define PB_FIELD_32BIT + * compile-time option. You can do that in pb.h or on compiler command line. + * + * The reason you need to do this is that some of your messages contain tag + * numbers or field sizes that are larger than what can fit in 8 or 16 bit + * field descriptors. + */ +PB_STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 65536 && pb_membersize(AllTypes, rep_submsg[0]) < 65536 && pb_membersize(AllTypes, opt_submsg) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_SubMessage_EmptyMessage_Limits_AllTypes) +#endif + +#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) +/* If you get an error here, it means that you need to define PB_FIELD_16BIT + * compile-time option. You can do that in pb.h or on compiler command line. + * + * The reason you need to do this is that some of your messages contain tag + * numbers or field sizes that are larger than what can fit in the default + * 8 bit descriptors. + */ +PB_STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 256 && pb_membersize(AllTypes, rep_submsg[0]) < 256 && pb_membersize(AllTypes, opt_submsg) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_SubMessage_EmptyMessage_Limits_AllTypes) +#endif + + +/* On some platforms (such as AVR), double is really float. + * These are not directly supported by nanopb, but see example_avr_double. + * To get rid of this error, remove any double fields from your .proto. + */ +PB_STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES) + diff --git a/tests/backwards_compatibility/alltypes_legacy.h b/tests/backwards_compatibility/alltypes_legacy.h index 037b347..4e0a63b 100644 --- a/tests/backwards_compatibility/alltypes_legacy.h +++ b/tests/backwards_compatibility/alltypes_legacy.h @@ -1,18 +1,24 @@ /* Automatically generated nanopb header */ -/* This is a file generated using nanopb-0.2.0-dev. - * It is used as a part of test suite in order to detect any - * incompatible changes made to the generator in future versions. - */ +/* Generated by nanopb-0.3.0-dev at Tue Aug 19 17:53:24 2014. */ -#ifndef _PB_ALLTYPES_PB_H_ -#define _PB_ALLTYPES_PB_H_ +#ifndef PB_ALLTYPES_LEGACY_H_INCLUDED +#define PB_ALLTYPES_LEGACY_H_INCLUDED #include <pb.h> +#if PB_PROTO_HEADER_VERSION != 30 +#error Regenerate this file with the current version of nanopb generator. +#endif + #ifdef __cplusplus extern "C" { #endif /* Enum definitions */ +typedef enum _HugeEnum { + HugeEnum_Negative = -2147483647, + HugeEnum_Positive = 2147483647 +} HugeEnum; + typedef enum _MyEnum { MyEnum_Zero = 0, MyEnum_First = 1, @@ -21,6 +27,23 @@ typedef enum _MyEnum { } MyEnum; /* Struct definitions */ +typedef struct _EmptyMessage { + uint8_t dummy_field; +} EmptyMessage; + +typedef struct _Limits { + int32_t int32_min; + int32_t int32_max; + uint32_t uint32_min; + uint32_t uint32_max; + int64_t int64_min; + int64_t int64_max; + uint64_t uint64_min; + uint64_t uint64_max; + HugeEnum enum_min; + HugeEnum enum_max; +} Limits; + typedef struct _SubMessage { char substuff1[16]; int32_t substuff2; @@ -28,20 +51,11 @@ typedef struct _SubMessage { uint32_t substuff3; } SubMessage; -typedef struct { - size_t size; - uint8_t bytes[16]; -} AllTypes_req_bytes_t; +typedef PB_BYTES_ARRAY_T(16) AllTypes_req_bytes_t; -typedef struct { - size_t size; - uint8_t bytes[16]; -} AllTypes_rep_bytes_t; +typedef PB_BYTES_ARRAY_T(16) AllTypes_rep_bytes_t; -typedef struct { - size_t size; - uint8_t bytes[16]; -} AllTypes_opt_bytes_t; +typedef PB_BYTES_ARRAY_T(16) AllTypes_opt_bytes_t; typedef struct _AllTypes { int32_t req_int32; @@ -61,39 +75,39 @@ typedef struct _AllTypes { AllTypes_req_bytes_t req_bytes; SubMessage req_submsg; MyEnum req_enum; - size_t rep_int32_count; + pb_size_t rep_int32_count; int32_t rep_int32[5]; - size_t rep_int64_count; + pb_size_t rep_int64_count; int64_t rep_int64[5]; - size_t rep_uint32_count; + pb_size_t rep_uint32_count; uint32_t rep_uint32[5]; - size_t rep_uint64_count; + pb_size_t rep_uint64_count; uint64_t rep_uint64[5]; - size_t rep_sint32_count; + pb_size_t rep_sint32_count; int32_t rep_sint32[5]; - size_t rep_sint64_count; + pb_size_t rep_sint64_count; int64_t rep_sint64[5]; - size_t rep_bool_count; + pb_size_t rep_bool_count; bool rep_bool[5]; - size_t rep_fixed32_count; + pb_size_t rep_fixed32_count; uint32_t rep_fixed32[5]; - size_t rep_sfixed32_count; + pb_size_t rep_sfixed32_count; int32_t rep_sfixed32[5]; - size_t rep_float_count; + pb_size_t rep_float_count; float rep_float[5]; - size_t rep_fixed64_count; + pb_size_t rep_fixed64_count; uint64_t rep_fixed64[5]; - size_t rep_sfixed64_count; + pb_size_t rep_sfixed64_count; int64_t rep_sfixed64[5]; - size_t rep_double_count; + pb_size_t rep_double_count; double rep_double[5]; - size_t rep_string_count; + pb_size_t rep_string_count; char rep_string[5][16]; - size_t rep_bytes_count; + pb_size_t rep_bytes_count; AllTypes_rep_bytes_t rep_bytes[5]; - size_t rep_submsg_count; + pb_size_t rep_submsg_count; SubMessage rep_submsg[5]; - size_t rep_enum_count; + pb_size_t rep_enum_count; MyEnum rep_enum[5]; bool has_opt_int32; int32_t opt_int32; @@ -130,12 +144,23 @@ typedef struct _AllTypes { bool has_opt_enum; MyEnum opt_enum; int32_t end; + pb_extension_t *extensions; } AllTypes; /* Default values for struct fields */ extern const char SubMessage_substuff1_default[16]; extern const int32_t SubMessage_substuff2_default; extern const uint32_t SubMessage_substuff3_default; +extern const int32_t Limits_int32_min_default; +extern const int32_t Limits_int32_max_default; +extern const uint32_t Limits_uint32_min_default; +extern const uint32_t Limits_uint32_max_default; +extern const int64_t Limits_int64_min_default; +extern const int64_t Limits_int64_max_default; +extern const uint64_t Limits_uint64_min_default; +extern const uint64_t Limits_uint64_max_default; +extern const HugeEnum Limits_enum_min_default; +extern const HugeEnum Limits_enum_max_default; extern const int32_t AllTypes_opt_int32_default; extern const int64_t AllTypes_opt_int64_default; extern const uint32_t AllTypes_opt_uint32_default; @@ -153,23 +178,94 @@ extern const char AllTypes_opt_string_default[16]; extern const AllTypes_opt_bytes_t AllTypes_opt_bytes_default; extern const MyEnum AllTypes_opt_enum_default; -/* Struct field encoding specification for nanopb */ -extern const pb_field_t SubMessage_fields[4]; -extern const pb_field_t AllTypes_fields[53]; +/* Initializer values for message structs */ +#define SubMessage_init_default {"1", 2, false, 3u} +#define EmptyMessage_init_default {0} +#define Limits_init_default {2147483647, -2147483647, 4294967295u, 0u, 9223372036854775807ll, -9223372036854775807ll, 18446744073709551615ull, 0ull, HugeEnum_Positive, HugeEnum_Negative} +#define AllTypes_init_default {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", {0, {0}}, SubMessage_init_default, (MyEnum)0, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {"", "", "", "", ""}, 0, {{0, {0}}, {0, {0}}, {0, {0}}, {0, {0}}, {0, {0}}}, 0, {SubMessage_init_default, SubMessage_init_default, SubMessage_init_default, SubMessage_init_default, SubMessage_init_default}, 0, {(MyEnum)0, (MyEnum)0, (MyEnum)0, (MyEnum)0, (MyEnum)0}, false, 4041, false, 4042ll, false, 4043u, false, 4044ull, false, 4045, false, 4046, false, false, false, 4048u, false, 4049, false, 4050, false, 4051ull, false, 4052ll, false, 4053, false, "4054", false, {4, {0x34,0x30,0x35,0x35}}, false, SubMessage_init_default, false, MyEnum_Second, 0, NULL} +#define SubMessage_init_zero {"", 0, false, 0} +#define EmptyMessage_init_zero {0} +#define Limits_init_zero {0, 0, 0, 0, 0, 0, 0, 0, (HugeEnum)0, (HugeEnum)0} +#define AllTypes_init_zero {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, "", {0, {0}}, SubMessage_init_zero, (MyEnum)0, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {0, 0, 0, 0, 0}, 0, {"", "", "", "", ""}, 0, {{0, {0}}, {0, {0}}, {0, {0}}, {0, {0}}, {0, {0}}}, 0, {SubMessage_init_zero, SubMessage_init_zero, SubMessage_init_zero, SubMessage_init_zero, SubMessage_init_zero}, 0, {(MyEnum)0, (MyEnum)0, (MyEnum)0, (MyEnum)0, (MyEnum)0}, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, 0, false, "", false, {0, {0}}, false, SubMessage_init_zero, false, (MyEnum)0, 0, NULL} -/* Check that field information fits in pb_field_t */ -#if !defined(PB_FIELD_16BIT) && !defined(PB_FIELD_32BIT) -STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 256 && pb_membersize(AllTypes, rep_submsg[0]) < 256 && pb_membersize(AllTypes, opt_submsg) < 256), YOU_MUST_DEFINE_PB_FIELD_16BIT_FOR_MESSAGES_SubMessage_AllTypes) -#endif +/* Field tags (for use in manual encoding/decoding) */ +#define Limits_int32_min_tag 1 +#define Limits_int32_max_tag 2 +#define Limits_uint32_min_tag 3 +#define Limits_uint32_max_tag 4 +#define Limits_int64_min_tag 5 +#define Limits_int64_max_tag 6 +#define Limits_uint64_min_tag 7 +#define Limits_uint64_max_tag 8 +#define Limits_enum_min_tag 9 +#define Limits_enum_max_tag 10 +#define SubMessage_substuff1_tag 1 +#define SubMessage_substuff2_tag 2 +#define SubMessage_substuff3_tag 3 +#define AllTypes_req_int32_tag 1 +#define AllTypes_req_int64_tag 2 +#define AllTypes_req_uint32_tag 3 +#define AllTypes_req_uint64_tag 4 +#define AllTypes_req_sint32_tag 5 +#define AllTypes_req_sint64_tag 6 +#define AllTypes_req_bool_tag 7 +#define AllTypes_req_fixed32_tag 8 +#define AllTypes_req_sfixed32_tag 9 +#define AllTypes_req_float_tag 10 +#define AllTypes_req_fixed64_tag 11 +#define AllTypes_req_sfixed64_tag 12 +#define AllTypes_req_double_tag 13 +#define AllTypes_req_string_tag 14 +#define AllTypes_req_bytes_tag 15 +#define AllTypes_req_submsg_tag 16 +#define AllTypes_req_enum_tag 17 +#define AllTypes_rep_int32_tag 21 +#define AllTypes_rep_int64_tag 22 +#define AllTypes_rep_uint32_tag 23 +#define AllTypes_rep_uint64_tag 24 +#define AllTypes_rep_sint32_tag 25 +#define AllTypes_rep_sint64_tag 26 +#define AllTypes_rep_bool_tag 27 +#define AllTypes_rep_fixed32_tag 28 +#define AllTypes_rep_sfixed32_tag 29 +#define AllTypes_rep_float_tag 30 +#define AllTypes_rep_fixed64_tag 31 +#define AllTypes_rep_sfixed64_tag 32 +#define AllTypes_rep_double_tag 33 +#define AllTypes_rep_string_tag 34 +#define AllTypes_rep_bytes_tag 35 +#define AllTypes_rep_submsg_tag 36 +#define AllTypes_rep_enum_tag 37 +#define AllTypes_opt_int32_tag 41 +#define AllTypes_opt_int64_tag 42 +#define AllTypes_opt_uint32_tag 43 +#define AllTypes_opt_uint64_tag 44 +#define AllTypes_opt_sint32_tag 45 +#define AllTypes_opt_sint64_tag 46 +#define AllTypes_opt_bool_tag 47 +#define AllTypes_opt_fixed32_tag 48 +#define AllTypes_opt_sfixed32_tag 49 +#define AllTypes_opt_float_tag 50 +#define AllTypes_opt_fixed64_tag 51 +#define AllTypes_opt_sfixed64_tag 52 +#define AllTypes_opt_double_tag 53 +#define AllTypes_opt_string_tag 54 +#define AllTypes_opt_bytes_tag 55 +#define AllTypes_opt_submsg_tag 56 +#define AllTypes_opt_enum_tag 57 +#define AllTypes_end_tag 99 -#if !defined(PB_FIELD_32BIT) -STATIC_ASSERT((pb_membersize(AllTypes, req_submsg) < 65536 && pb_membersize(AllTypes, rep_submsg[0]) < 65536 && pb_membersize(AllTypes, opt_submsg) < 65536), YOU_MUST_DEFINE_PB_FIELD_32BIT_FOR_MESSAGES_SubMessage_AllTypes) -#endif +/* Struct field encoding specification for nanopb */ +extern const pb_field_t SubMessage_fields[4]; +extern const pb_field_t EmptyMessage_fields[1]; +extern const pb_field_t Limits_fields[11]; +extern const pb_field_t AllTypes_fields[54]; -/* On some platforms (such as AVR), double is really float. - * These are not directly supported by nanopb, but see example_avr_double. - */ -STATIC_ASSERT(sizeof(double) == 8, DOUBLE_MUST_BE_8_BYTES) +/* Maximum encoded size of messages (where known) */ +#define SubMessage_size 34 +#define EmptyMessage_size 0 +#define Limits_size 90 +#define AllTypes_size 1362 #ifdef __cplusplus } /* extern "C" */ diff --git a/tests/backwards_compatibility/alltypes_legacy.options b/tests/backwards_compatibility/alltypes_legacy.options new file mode 100644 index 0000000..b31e3cf --- /dev/null +++ b/tests/backwards_compatibility/alltypes_legacy.options @@ -0,0 +1,3 @@ +* max_size:16 +* max_count:5 + diff --git a/tests/backwards_compatibility/alltypes_legacy.proto b/tests/backwards_compatibility/alltypes_legacy.proto new file mode 100644 index 0000000..f5bc35c --- /dev/null +++ b/tests/backwards_compatibility/alltypes_legacy.proto @@ -0,0 +1,110 @@ +syntax = "proto2"; + +message SubMessage { + required string substuff1 = 1 [default = "1"]; + required int32 substuff2 = 2 [default = 2]; + optional fixed32 substuff3 = 3 [default = 3]; +} + +message EmptyMessage { + +} + +enum HugeEnum { + Negative = -2147483647; /* protoc doesn't accept -2147483648 here */ + Positive = 2147483647; +} + +message Limits { + required int32 int32_min = 1 [default = 2147483647]; + required int32 int32_max = 2 [default = -2147483647]; + required uint32 uint32_min = 3 [default = 4294967295]; + required uint32 uint32_max = 4 [default = 0]; + required int64 int64_min = 5 [default = 9223372036854775807]; + required int64 int64_max = 6 [default = -9223372036854775807]; + required uint64 uint64_min = 7 [default = 18446744073709551615]; + required uint64 uint64_max = 8 [default = 0]; + required HugeEnum enum_min = 9 [default = Positive]; + required HugeEnum enum_max = 10 [default = Negative]; +} + +enum MyEnum { + Zero = 0; + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + required int32 req_int32 = 1; + required int64 req_int64 = 2; + required uint32 req_uint32 = 3; + required uint64 req_uint64 = 4; + required sint32 req_sint32 = 5; + required sint64 req_sint64 = 6; + required bool req_bool = 7; + + required fixed32 req_fixed32 = 8; + required sfixed32 req_sfixed32= 9; + required float req_float = 10; + + required fixed64 req_fixed64 = 11; + required sfixed64 req_sfixed64= 12; + required double req_double = 13; + + required string req_string = 14; + required bytes req_bytes = 15; + required SubMessage req_submsg = 16; + required MyEnum req_enum = 17; + + + repeated int32 rep_int32 = 21 [packed = true]; + repeated int64 rep_int64 = 22 [packed = true]; + repeated uint32 rep_uint32 = 23 [packed = true]; + repeated uint64 rep_uint64 = 24 [packed = true]; + repeated sint32 rep_sint32 = 25 [packed = true]; + repeated sint64 rep_sint64 = 26 [packed = true]; + repeated bool rep_bool = 27 [packed = true]; + + repeated fixed32 rep_fixed32 = 28 [packed = true]; + repeated sfixed32 rep_sfixed32= 29 [packed = true]; + repeated float rep_float = 30 [packed = true]; + + repeated fixed64 rep_fixed64 = 31 [packed = true]; + repeated sfixed64 rep_sfixed64= 32 [packed = true]; + repeated double rep_double = 33 [packed = true]; + + repeated string rep_string = 34; + repeated bytes rep_bytes = 35; + repeated SubMessage rep_submsg = 36; + repeated MyEnum rep_enum = 37 [packed = true]; + + optional int32 opt_int32 = 41 [default = 4041]; + optional int64 opt_int64 = 42 [default = 4042]; + optional uint32 opt_uint32 = 43 [default = 4043]; + optional uint64 opt_uint64 = 44 [default = 4044]; + optional sint32 opt_sint32 = 45 [default = 4045]; + optional sint64 opt_sint64 = 46 [default = 4046]; + optional bool opt_bool = 47 [default = false]; + + optional fixed32 opt_fixed32 = 48 [default = 4048]; + optional sfixed32 opt_sfixed32= 49 [default = 4049]; + optional float opt_float = 50 [default = 4050]; + + optional fixed64 opt_fixed64 = 51 [default = 4051]; + optional sfixed64 opt_sfixed64= 52 [default = 4052]; + optional double opt_double = 53 [default = 4053]; + + optional string opt_string = 54 [default = "4054"]; + optional bytes opt_bytes = 55 [default = "4055"]; + optional SubMessage opt_submsg = 56; + optional MyEnum opt_enum = 57 [default = Second]; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + required int32 end = 99; + + + extensions 200 to 255; +} + diff --git a/tests/backwards_compatibility/decode_legacy.c b/tests/backwards_compatibility/decode_legacy.c index 9dfe437..5f5b6bb 100644 --- a/tests/backwards_compatibility/decode_legacy.c +++ b/tests/backwards_compatibility/decode_legacy.c @@ -22,10 +22,7 @@ the decoding and checks the fields. */ bool check_alltypes(pb_istream_t *stream, int mode) { - AllTypes alltypes; - - /* Fill with garbage to better detect initialization errors */ - memset(&alltypes, 0xAA, sizeof(alltypes)); + AllTypes alltypes = {0}; if (!pb_decode(stream, AllTypes_fields, &alltypes)) return false; diff --git a/tests/basic_buffer/SConscript b/tests/basic_buffer/SConscript index 2546aaa..acaf5ff 100644 --- a/tests/basic_buffer/SConscript +++ b/tests/basic_buffer/SConscript @@ -2,8 +2,8 @@ Import("env") -enc = env.Program(["encode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_buffer.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(enc) env.RunTest([dec, "encode_buffer.output"]) diff --git a/tests/basic_buffer/decode_buffer.c b/tests/basic_buffer/decode_buffer.c index fae9e2f..291d164 100644 --- a/tests/basic_buffer/decode_buffer.c +++ b/tests/basic_buffer/decode_buffer.c @@ -16,7 +16,7 @@ bool print_person(pb_istream_t *stream) { int i; - Person person; + Person person = Person_init_zero; if (!pb_decode(stream, Person_fields, &person)) return false; diff --git a/tests/basic_stream/SConscript b/tests/basic_stream/SConscript index 46db8c4..7d66856 100644 --- a/tests/basic_stream/SConscript +++ b/tests/basic_stream/SConscript @@ -2,8 +2,8 @@ Import("env") -enc = env.Program(["encode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_stream.c", "$COMMON/person.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(enc) env.RunTest([dec, "encode_stream.output"]) diff --git a/tests/basic_stream/decode_stream.c b/tests/basic_stream/decode_stream.c index 667bf3c..798dcc5 100644 --- a/tests/basic_stream/decode_stream.c +++ b/tests/basic_stream/decode_stream.c @@ -12,7 +12,7 @@ bool print_person(pb_istream_t *stream) { int i; - Person person; + Person person = Person_init_zero; if (!pb_decode(stream, Person_fields, &person)) return false; diff --git a/tests/buffer_only/SConscript b/tests/buffer_only/SConscript index cddbb04..55b747b 100644 --- a/tests/buffer_only/SConscript +++ b/tests/buffer_only/SConscript @@ -18,10 +18,11 @@ strict = opts.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode_bufonly.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode_bufonly.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_bufonly.o", "$NANOPB/pb_common.c") # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_bufonly.o"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_bufonly.o"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_bufonly.o", "pb_common_bufonly.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_bufonly.o", "pb_common_bufonly.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/callbacks/SConscript b/tests/callbacks/SConscript index 9ec8a43..4452143 100644 --- a/tests/callbacks/SConscript +++ b/tests/callbacks/SConscript @@ -3,8 +3,8 @@ Import("env") env.NanopbProto("callbacks") -enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_encode.o"]) -dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_decode.o"]) +enc = env.Program(["encode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = env.Program(["decode_callbacks.c", "callbacks.pb.c", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(enc) env.RunTest([dec, "encode_callbacks.output"]) diff --git a/tests/callbacks/callbacks.proto b/tests/callbacks/callbacks.proto index ccd1edd..96ac744 100644 --- a/tests/callbacks/callbacks.proto +++ b/tests/callbacks/callbacks.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message SubMessage { optional string stringvalue = 1; repeated int32 int32value = 2; diff --git a/tests/common/SConscript b/tests/common/SConscript index 144f149..05e2f85 100644 --- a/tests/common/SConscript +++ b/tests/common/SConscript @@ -8,10 +8,41 @@ env.NanopbProto("unittestproto") # Protocol definitions for basic_buffer/stream tests env.NanopbProto("person") +#-------------------------------------------- # Binaries of the pb_decode.c and pb_encode.c # These are built using more strict warning flags. strict = env.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common.o", "$NANOPB/pb_common.c") + +#----------------------------------------------- +# Binaries of pb_decode etc. with malloc support +# Uses malloc_wrappers.c to count allocations. +malloc_env = env.Clone() +malloc_env.Append(CPPDEFINES = {'PB_ENABLE_MALLOC': 1, + 'PB_SYSTEM_HEADER': '\\"malloc_wrappers_syshdr.h\\"'}) +malloc_env.Append(CPPPATH = ["$COMMON"]) + +if 'SYSHDR' in malloc_env: + malloc_env.Append(CPPDEFINES = {'PB_OLD_SYSHDR': malloc_env['SYSHDR']}) + +# Disable libmudflap, because it will confuse valgrind +# and other memory leak detection tools. +if '-fmudflap' in env["CCFLAGS"]: + malloc_env["CCFLAGS"].remove("-fmudflap") + malloc_env["LINKFLAGS"].remove("-fmudflap") + malloc_env["LIBS"].remove("mudflap") + +malloc_strict = malloc_env.Clone() +malloc_strict.Append(CFLAGS = malloc_strict['CORECFLAGS']) +malloc_strict.Object("pb_decode_with_malloc.o", "$NANOPB/pb_decode.c") +malloc_strict.Object("pb_encode_with_malloc.o", "$NANOPB/pb_encode.c") +malloc_strict.Object("pb_common_with_malloc.o", "$NANOPB/pb_common.c") + +malloc_env.Object("malloc_wrappers.o", "malloc_wrappers.c") +malloc_env.Depends("$NANOPB/pb.h", ["malloc_wrappers_syshdr.h", "malloc_wrappers.h"]) + +Export("malloc_env") diff --git a/tests/common/malloc_wrappers.c b/tests/common/malloc_wrappers.c new file mode 100644 index 0000000..ad69f1c --- /dev/null +++ b/tests/common/malloc_wrappers.c @@ -0,0 +1,54 @@ +#include "malloc_wrappers.h" +#include <stdint.h> +#include <assert.h> +#include <string.h> + +static size_t alloc_count = 0; + +/* Allocate memory and place check values before and after. */ +void* malloc_with_check(size_t size) +{ + size_t size32 = (size + 3) / 4 + 3; + uint32_t *buf = malloc(size32 * sizeof(uint32_t)); + buf[0] = size32; + buf[1] = 0xDEADBEEF; + buf[size32 - 1] = 0xBADBAD; + return buf + 2; +} + +/* Free memory allocated with malloc_with_check() and do the checks. */ +void free_with_check(void *mem) +{ + uint32_t *buf = (uint32_t*)mem - 2; + assert(buf[1] == 0xDEADBEEF); + assert(buf[buf[0] - 1] == 0xBADBAD); + free(buf); +} + +/* Track memory usage */ +void* counting_realloc(void *ptr, size_t size) +{ + /* Don't allocate crazy amounts of RAM when fuzzing */ + if (size > 1000000) + return NULL; + + if (!ptr && size) + alloc_count++; + + return realloc(ptr, size); +} + +void counting_free(void *ptr) +{ + if (ptr) + { + assert(alloc_count > 0); + alloc_count--; + free(ptr); + } +} + +size_t get_alloc_count() +{ + return alloc_count; +} diff --git a/tests/common/malloc_wrappers.h b/tests/common/malloc_wrappers.h new file mode 100644 index 0000000..7eec795 --- /dev/null +++ b/tests/common/malloc_wrappers.h @@ -0,0 +1,7 @@ +#include <stdlib.h> + +void* malloc_with_check(size_t size); +void free_with_check(void *mem); +void* counting_realloc(void *ptr, size_t size); +void counting_free(void *ptr); +size_t get_alloc_count(); diff --git a/tests/common/malloc_wrappers_syshdr.h b/tests/common/malloc_wrappers_syshdr.h new file mode 100644 index 0000000..d295d9e --- /dev/null +++ b/tests/common/malloc_wrappers_syshdr.h @@ -0,0 +1,15 @@ +/* This is just a wrapper in order to get our own malloc wrappers into nanopb core. */ + +#define pb_realloc(ptr,size) counting_realloc(ptr,size) +#define pb_free(ptr) counting_free(ptr) + +#ifdef PB_OLD_SYSHDR +#include PB_OLD_SYSHDR +#else +#include <stdint.h> +#include <stddef.h> +#include <stdbool.h> +#include <string.h> +#endif + +#include <malloc_wrappers.h> diff --git a/tests/common/person.proto b/tests/common/person.proto index dafcf93..becefdf 100644 --- a/tests/common/person.proto +++ b/tests/common/person.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import "nanopb.proto"; message Person { diff --git a/tests/common/unittestproto.proto b/tests/common/unittestproto.proto index eb3e7de..23b5b97 100644 --- a/tests/common/unittestproto.proto +++ b/tests/common/unittestproto.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import 'nanopb.proto'; message IntegerArray { @@ -34,3 +36,8 @@ message CallbackContainer { message CallbackContainerContainer { required CallbackContainer submsg = 1; } + +message StringPointerContainer { + repeated string rep_str = 1 [(nanopb).type = FT_POINTER]; +} + diff --git a/tests/cxx_main_program/SConscript b/tests/cxx_main_program/SConscript index e78c6b3..edb8812 100644 --- a/tests/cxx_main_program/SConscript +++ b/tests/cxx_main_program/SConscript @@ -11,14 +11,15 @@ env.Append(CPPDEFINES = ['__STDC_LIMIT_MACROS']) c = Copy("$TARGET", "$SOURCE") env.Command("pb_encode.cxx", "#../pb_encode.c", c) env.Command("pb_decode.cxx", "#../pb_decode.c", c) +env.Command("pb_common.cxx", "#../pb_common.c", c) env.Command("alltypes.pb.h", "$BUILD/alltypes/alltypes.pb.h", c) env.Command("alltypes.pb.cxx", "$BUILD/alltypes/alltypes.pb.c", c) env.Command("encode_alltypes.cxx", "$BUILD/alltypes/encode_alltypes.c", c) env.Command("decode_alltypes.cxx", "$BUILD/alltypes/decode_alltypes.c", c) # Now build and run the test normally. -enc = env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx"]) -dec = env.Program(["decode_alltypes.cxx", "alltypes.pb.cxx", "pb_decode.cxx"]) +enc = env.Program(["encode_alltypes.cxx", "alltypes.pb.cxx", "pb_encode.cxx", "pb_common.cxx"]) +dec = env.Program(["decode_alltypes.cxx", "alltypes.pb.cxx", "pb_decode.cxx", "pb_common.cxx"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/cyclic_messages/SConscript b/tests/cyclic_messages/SConscript new file mode 100644 index 0000000..c782001 --- /dev/null +++ b/tests/cyclic_messages/SConscript @@ -0,0 +1,11 @@ +Import("env") + +# Encode cyclic messages with callback fields + +c = Copy("$TARGET", "$SOURCE") +env.Command("cyclic_callback.proto", "cyclic.proto", c) +env.NanopbProto(["cyclic_callback", "cyclic_callback.options"]) + +enc_callback = env.Program(["encode_cyclic_callback.c", "cyclic_callback.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) + + diff --git a/tests/cyclic_messages/cyclic.proto b/tests/cyclic_messages/cyclic.proto new file mode 100644 index 0000000..8cab0b1 --- /dev/null +++ b/tests/cyclic_messages/cyclic.proto @@ -0,0 +1,27 @@ +// Test structures with cyclic references. +// These can only be handled in pointer/callback mode, +// see associated .options files. + +syntax = "proto2"; + +message TreeNode +{ + optional int32 leaf = 1; + optional TreeNode left = 2; + optional TreeNode right = 3; +} + +message Dictionary +{ + repeated KeyValuePair dictItem = 1; +} + +message KeyValuePair +{ + required string key = 1; + optional string stringValue = 2; + optional int32 intValue = 3; + optional Dictionary dictValue = 4; + optional TreeNode treeValue = 5; +} + diff --git a/tests/cyclic_messages/cyclic_callback.options b/tests/cyclic_messages/cyclic_callback.options new file mode 100644 index 0000000..fd4e1e1 --- /dev/null +++ b/tests/cyclic_messages/cyclic_callback.options @@ -0,0 +1,7 @@ +TreeNode.left type:FT_CALLBACK +TreeNode.right type:FT_CALLBACK + +Dictionary.data type:FT_CALLBACK +KeyValuePair.key max_size:8 +KeyValuePair.stringValue max_size:8 +KeyValuePair.treeValue type:FT_CALLBACK diff --git a/tests/cyclic_messages/encode_cyclic_callback.c b/tests/cyclic_messages/encode_cyclic_callback.c new file mode 100644 index 0000000..7f67e70 --- /dev/null +++ b/tests/cyclic_messages/encode_cyclic_callback.c @@ -0,0 +1,148 @@ +/* This program parses an input string in a format a bit like JSON: + * {'foobar': 1234, 'xyz': 'abc', 'tree': [[[1, 2], 3], [4, 5]]} + * and encodes it as protobuf + * + * Note: The string parsing here is not in any way intended to be robust + * nor safe against buffer overflows. It is just for this test. + */ + +#include <pb_encode.h> +#include <stdlib.h> +#include <stdio.h> +#include <string.h> +#include "cyclic_callback.pb.h" + +static char *find_end_of_item(char *p) +{ + int depth = 0; + do { + if (*p == '[' || *p == '{') depth++; + if (*p == ']' || *p == '}') depth--; + p++; + } while (depth > 0 || (*p != ',' && *p != '}')); + + if (*p == '}') + return p; /* End of parent dict */ + + p++; + while (*p == ' ') p++; + return p; +} + +/* Parse a tree in format [[1 2] 3] and encode it directly to protobuf */ +static bool encode_tree(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + TreeNode tree = TreeNode_init_zero; + char *p = (char*)*arg; + + if (*p == '[') + { + /* This is a tree branch */ + p++; + tree.left.funcs.encode = encode_tree; + tree.left.arg = p; + + p = find_end_of_item(p); + tree.right.funcs.encode = encode_tree; + tree.right.arg = p; + } + else + { + /* This is a leaf node */ + tree.has_leaf = true; + tree.leaf = atoi(p); + } + + return pb_encode_tag_for_field(stream, field) && + pb_encode_submessage(stream, TreeNode_fields, &tree); +} + +/* Parse a dictionary in format {'name': value} and encode it directly to protobuf */ +static bool encode_dictionary(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + int textlen; + char *p = (char*)*arg; + if (*p == '{') p++; + while (*p != '}') + { + KeyValuePair pair = KeyValuePair_init_zero; + + if (*p != '\'') + PB_RETURN_ERROR(stream, "invalid key, missing quote"); + + p++; /* Starting quote of key */ + textlen = strchr(p, '\'') - p; + strncpy(pair.key, p, textlen); + pair.key[textlen] = 0; + p += textlen + 2; + + while (*p == ' ') p++; + + if (*p == '[') + { + /* Value is a tree */ + pair.treeValue.funcs.encode = encode_tree; + pair.treeValue.arg = p; + } + else if (*p == '\'') + { + /* Value is a string */ + pair.has_stringValue = true; + p++; + textlen = strchr(p, '\'') - p; + strncpy(pair.stringValue, p, textlen); + pair.stringValue[textlen] = 0; + } + else if (*p == '{') + { + /* Value is a dictionary */ + pair.has_dictValue = true; + pair.dictValue.dictItem.funcs.encode = encode_dictionary; + pair.dictValue.dictItem.arg = p; + } + else + { + /* Value is integer */ + pair.has_intValue = true; + pair.intValue = atoi(p); + } + + p = find_end_of_item(p); + + if (!pb_encode_tag_for_field(stream, field)) + return false; + + if (!pb_encode_submessage(stream, KeyValuePair_fields, &pair)) + return false; + } + + return true; +} + + +int main(int argc, char *argv[]) +{ + uint8_t buffer[256]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + Dictionary dict = Dictionary_init_zero; + + if (argc <= 1) + { + fprintf(stderr, "Usage: %s \"{'foobar': 1234, ...}\"\n", argv[0]); + return 1; + } + + dict.dictItem.funcs.encode = encode_dictionary; + dict.dictItem.arg = argv[1]; + + if (!pb_encode(&stream, Dictionary_fields, &dict)) + { + fprintf(stderr, "Encoding error: %s\n", PB_GET_ERROR(&stream)); + return 1; + } + + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; +} + + diff --git a/tests/decode_unittests/decode_unittests.c b/tests/decode_unittests/decode_unittests.c index 1be0191..dc1e719 100644 --- a/tests/decode_unittests/decode_unittests.c +++ b/tests/decode_unittests/decode_unittests.c @@ -1,4 +1,6 @@ /* This includes the whole .c file to get access to static functions. */ +#define PB_ENABLE_MALLOC +#include "pb_common.c" #include "pb_decode.c" #include <stdio.h> @@ -85,6 +87,23 @@ int main() pb_decode_varint(&s, (uint64_t*)&i) && i == -1)); TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_decode_varint(&s, &u) && u == UINT64_MAX)); + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), + !pb_decode_varint(&s, &u))); + } + + { + pb_istream_t s; + uint32_t u; + + COMMENT("Test pb_decode_varint32"); + TEST((s = S("\x00"), pb_decode_varint32(&s, &u) && u == 0)); + TEST((s = S("\x01"), pb_decode_varint32(&s, &u) && u == 1)); + TEST((s = S("\xAC\x02"), pb_decode_varint32(&s, &u) && u == 300)); + TEST((s = S("\xFF\xFF\xFF\xFF\x0F"), pb_decode_varint32(&s, &u) && u == UINT32_MAX)); + TEST((s = S("\xFF\xFF\xFF\xFF\x8F\x00"), pb_decode_varint32(&s, &u) && u == UINT32_MAX)); + TEST((s = S("\xFF\xFF\xFF\xFF\x10"), !pb_decode_varint32(&s, &u))); + TEST((s = S("\xFF\xFF\xFF\xFF\x40"), !pb_decode_varint32(&s, &u))); + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\x01"), !pb_decode_varint32(&s, &u))); } { @@ -107,16 +126,16 @@ int main() } { - pb_istream_t s = S("\x01\xFF\xFF\x03"); + pb_istream_t s = S("\x01\x00"); pb_field_t f = {1, PB_LTYPE_VARINT, 0, 0, 4, 0, 0}; uint32_t d; COMMENT("Test pb_dec_varint using uint32_t") TEST(pb_dec_varint(&s, &f, &d) && d == 1) /* Verify that no more than data_size is written. */ - d = 0; + d = 0xFFFFFFFF; f.data_size = 1; - TEST(pb_dec_varint(&s, &f, &d) && (d == 0xFF || d == 0xFF000000)) + TEST(pb_dec_varint(&s, &f, &d) && (d == 0xFFFFFF00 || d == 0x00FFFFFF)) } { @@ -134,9 +153,9 @@ int main() { pb_istream_t s; pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 8, 0, 0}; - uint64_t d; + int64_t d; - COMMENT("Test pb_dec_svarint using uint64_t") + COMMENT("Test pb_dec_svarint using int64_t") TEST((s = S("\x01"), pb_dec_svarint(&s, &f, &d) && d == -1)) TEST((s = S("\x02"), pb_dec_svarint(&s, &f, &d) && d == 1)) TEST((s = S("\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_svarint(&s, &f, &d) && d == INT64_MAX)) @@ -145,6 +164,50 @@ int main() { pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 4, 0, 0}; + int32_t d; + + COMMENT("Test pb_dec_svarint overflow detection using int32_t"); + TEST((s = S("\xfe\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d))); + TEST((s = S("\xfe\xff\xff\xff\x10"), !pb_dec_svarint(&s, &f, &d))); + TEST((s = S("\xff\xff\xff\xff\x0f"), pb_dec_svarint(&s, &f, &d))); + TEST((s = S("\xff\xff\xff\xff\x10"), !pb_dec_svarint(&s, &f, &d))); + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 4, 0, 0}; + uint32_t d; + + COMMENT("Test pb_dec_uvarint using uint32_t") + TEST((s = S("\x01"), pb_dec_uvarint(&s, &f, &d) && d == 1)) + TEST((s = S("\x02"), pb_dec_uvarint(&s, &f, &d) && d == 2)) + TEST((s = S("\xff\xff\xff\xff\x0f"), pb_dec_uvarint(&s, &f, &d) && d == UINT32_MAX)) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 8, 0, 0}; + uint64_t d; + + COMMENT("Test pb_dec_uvarint using uint64_t") + TEST((s = S("\x01"), pb_dec_uvarint(&s, &f, &d) && d == 1)) + TEST((s = S("\x02"), pb_dec_uvarint(&s, &f, &d) && d == 2)) + TEST((s = S("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF\x01"), pb_dec_uvarint(&s, &f, &d) && d == UINT64_MAX)) + } + + { + pb_istream_t s; + pb_field_t f = {1, PB_LTYPE_SVARINT, 0, 0, 4, 0, 0}; + uint32_t d; + + COMMENT("Test pb_dec_uvarint overflow detection using int32_t"); + TEST((s = S("\xff\xff\xff\xff\x0f"), pb_dec_uvarint(&s, &f, &d))); + TEST((s = S("\xff\xff\xff\xff\x10"), !pb_dec_uvarint(&s, &f, &d))); + } + + { + pb_istream_t s; pb_field_t f = {1, PB_LTYPE_FIXED32, 0, 0, 4, 0, 0}; float d; @@ -168,7 +231,7 @@ int main() { pb_istream_t s; - struct { size_t size; uint8_t bytes[5]; } d; + struct { pb_size_t size; uint8_t bytes[5]; } d; pb_field_t f = {1, PB_LTYPE_BYTES, 0, 0, sizeof(d), 0, 0}; COMMENT("Test pb_dec_bytes") @@ -249,7 +312,7 @@ int main() { pb_istream_t s; CallbackArray dest; - struct { size_t size; uint8_t bytes[10]; } ref; + struct { pb_size_t size; uint8_t bytes[10]; } ref; dest.data.funcs.decode = &callback_check; dest.data.arg = &ref; @@ -291,6 +354,14 @@ int main() { pb_istream_t s; + IntegerArray dest; + + COMMENT("Testing pb_decode with invalid tag numbers") + TEST((s = S("\x9f\xea"), !pb_decode(&s, IntegerArray_fields, &dest))); + } + + { + pb_istream_t s; IntegerContainer dest = {{0}}; COMMENT("Testing pb_decode_delimited") @@ -299,6 +370,28 @@ int main() dest.submsg.data_count == 5) } + { + pb_istream_t s = {0}; + void *data = NULL; + + COMMENT("Testing allocate_field") + TEST(allocate_field(&s, &data, 10, 10) && data != NULL); + TEST(allocate_field(&s, &data, 10, 20) && data != NULL); + + { + void *oldvalue = data; + size_t very_big = (size_t)-1; + size_t somewhat_big = very_big / 2 + 1; + size_t not_so_big = (size_t)1 << (4 * sizeof(size_t)); + + TEST(!allocate_field(&s, &data, very_big, 2) && data == oldvalue); + TEST(!allocate_field(&s, &data, somewhat_big, 2) && data == oldvalue); + TEST(!allocate_field(&s, &data, not_so_big, not_so_big) && data == oldvalue); + } + + pb_free(data); + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); diff --git a/tests/encode_unittests/encode_unittests.c b/tests/encode_unittests/encode_unittests.c index 06935f9..583af5c 100644 --- a/tests/encode_unittests/encode_unittests.c +++ b/tests/encode_unittests/encode_unittests.c @@ -1,4 +1,5 @@ /* This includes the whole .c file to get access to static functions. */ +#include "pb_common.c" #include "pb_encode.c" #include <stdio.h> @@ -169,7 +170,7 @@ int main() { uint8_t buffer[30]; pb_ostream_t s; - struct { size_t size; uint8_t bytes[5]; } value = {5, {'x', 'y', 'z', 'z', 'y'}}; + struct { pb_size_t size; uint8_t bytes[5]; } value = {5, {'x', 'y', 'z', 'z', 'y'}}; COMMENT("Test pb_enc_bytes") TEST(WRITES(pb_enc_bytes(&s, &BytesMessage_fields[0], &value), "\x05xyzzy")) @@ -330,6 +331,23 @@ int main() TEST(s.bytes_written == StringMessage_size); } + { + uint8_t buffer[128]; + pb_ostream_t s; + StringPointerContainer msg = StringPointerContainer_init_zero; + char *strs[1] = {NULL}; + char zstr[] = "Z"; + + COMMENT("Test string pointer encoding."); + + msg.rep_str = strs; + msg.rep_str_count = 1; + TEST(WRITES(pb_encode(&s, StringPointerContainer_fields, &msg), "\x0a\x00")) + + strs[0] = zstr; + TEST(WRITES(pb_encode(&s, StringPointerContainer_fields, &msg), "\x0a\x01Z")) + } + if (status != 0) fprintf(stdout, "\n\nSome tests FAILED!\n"); diff --git a/tests/enum_sizes/SConscript b/tests/enum_sizes/SConscript new file mode 100644 index 0000000..048592e --- /dev/null +++ b/tests/enum_sizes/SConscript @@ -0,0 +1,12 @@ +# Test that different sizes of enum fields are properly encoded and decoded. + +Import('env') + +env.NanopbProto('enumsizes') + +p = env.Program(["enumsizes_unittests.c", + "enumsizes.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_decode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) diff --git a/tests/enum_sizes/enumsizes.proto b/tests/enum_sizes/enumsizes.proto new file mode 100644 index 0000000..a85d416 --- /dev/null +++ b/tests/enum_sizes/enumsizes.proto @@ -0,0 +1,86 @@ +/* Test handling of enums with different value ranges. + * Depending on compiler and the packed_enum setting, the datatypes + * for enums can be either signed or unsigned. In past this has caused + * a bit of a problem for the encoder/decoder (issue #164). + */ + +syntax = "proto2"; + +import 'nanopb.proto'; + +option (nanopb_fileopt).long_names = false; + +enum UnpackedUint8 { + option (nanopb_enumopt).packed_enum = false; + UU8_MIN = 0; + UU8_MAX = 255; +} + +enum PackedUint8 { + option (nanopb_enumopt).packed_enum = true; + PU8_MIN = 0; + PU8_MAX = 255; +} + +enum UnpackedInt8 { + option (nanopb_enumopt).packed_enum = false; + UI8_MIN = -128; + UI8_MAX = 127; +} + +enum PackedInt8 { + option (nanopb_enumopt).packed_enum = true; + PI8_MIN = -128; + PI8_MAX = 127; +} + +enum UnpackedUint16 { + option (nanopb_enumopt).packed_enum = false; + UU16_MIN = 0; + UU16_MAX = 65535; +} + +enum PackedUint16 { + option (nanopb_enumopt).packed_enum = true; + PU16_MIN = 0; + PU16_MAX = 65535; +} + +enum UnpackedInt16 { + option (nanopb_enumopt).packed_enum = false; + UI16_MIN = -32768; + UI16_MAX = 32767; +} + +enum PackedInt16 { + option (nanopb_enumopt).packed_enum = true; + PI16_MIN = -32768; + PI16_MAX = 32767; +} + +/* Protobuf supports enums up to 32 bits. + * The 32 bit case is covered by HugeEnum in the alltypes test. + */ + +message PackedEnums { + required PackedUint8 u8_min = 1; + required PackedUint8 u8_max = 2; + required PackedInt8 i8_min = 3; + required PackedInt8 i8_max = 4; + required PackedUint16 u16_min = 5; + required PackedUint16 u16_max = 6; + required PackedInt16 i16_min = 7; + required PackedInt16 i16_max = 8; +} + +message UnpackedEnums { + required UnpackedUint8 u8_min = 1; + required UnpackedUint8 u8_max = 2; + required UnpackedInt8 i8_min = 3; + required UnpackedInt8 i8_max = 4; + required UnpackedUint16 u16_min = 5; + required UnpackedUint16 u16_max = 6; + required UnpackedInt16 i16_min = 7; + required UnpackedInt16 i16_max = 8; +} + diff --git a/tests/enum_sizes/enumsizes_unittests.c b/tests/enum_sizes/enumsizes_unittests.c new file mode 100644 index 0000000..5606895 --- /dev/null +++ b/tests/enum_sizes/enumsizes_unittests.c @@ -0,0 +1,72 @@ +#include <stdio.h> +#include <string.h> +#include <pb_decode.h> +#include <pb_encode.h> +#include "unittests.h" +#include "enumsizes.pb.h" + +int main() +{ + int status = 0; + + UnpackedEnums msg1 = { + UU8_MIN, UU8_MAX, + UI8_MIN, UI8_MAX, + UU16_MIN, UU16_MAX, + UI16_MIN, UI16_MAX, + }; + + PackedEnums msg2; + UnpackedEnums msg3; + uint8_t buf[256]; + size_t msgsize; + + COMMENT("Step 1: unpacked enums -> protobuf"); + { + pb_ostream_t s = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&s, UnpackedEnums_fields, &msg1)); + msgsize = s.bytes_written; + } + + COMMENT("Step 2: protobuf -> packed enums"); + { + pb_istream_t s = pb_istream_from_buffer(buf, msgsize); + TEST(pb_decode(&s, PackedEnums_fields, &msg2)); + + TEST(msg1.u8_min == (int)msg2.u8_min); + TEST(msg1.u8_max == (int)msg2.u8_max); + TEST(msg1.i8_min == (int)msg2.i8_min); + TEST(msg1.i8_max == (int)msg2.i8_max); + TEST(msg1.u16_min == (int)msg2.u16_min); + TEST(msg1.u16_max == (int)msg2.u16_max); + TEST(msg1.i16_min == (int)msg2.i16_min); + TEST(msg1.i16_max == (int)msg2.i16_max); + } + + COMMENT("Step 3: packed enums -> protobuf"); + { + pb_ostream_t s = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&s, PackedEnums_fields, &msg2)); + msgsize = s.bytes_written; + } + + COMMENT("Step 4: protobuf -> unpacked enums"); + { + pb_istream_t s = pb_istream_from_buffer(buf, msgsize); + TEST(pb_decode(&s, UnpackedEnums_fields, &msg3)); + + TEST(msg1.u8_min == (int)msg3.u8_min); + TEST(msg1.u8_max == (int)msg3.u8_max); + TEST(msg1.i8_min == (int)msg3.i8_min); + TEST(msg1.i8_max == (int)msg3.i8_max); + TEST(msg1.u16_min == (int)msg2.u16_min); + TEST(msg1.u16_max == (int)msg2.u16_max); + TEST(msg1.i16_min == (int)msg2.i16_min); + TEST(msg1.i16_max == (int)msg2.i16_max); + } + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} diff --git a/tests/enum_to_string/SConscript b/tests/enum_to_string/SConscript new file mode 100644 index 0000000..e86fcca --- /dev/null +++ b/tests/enum_to_string/SConscript @@ -0,0 +1,7 @@ +# Test enum to string functionality + +Import('env') +env.NanopbProto("enum.proto") +p = env.Program(["enum_to_string.c", "enum.pb.c"]) +env.RunTest(p) + diff --git a/tests/enum_to_string/enum.proto b/tests/enum_to_string/enum.proto new file mode 100644 index 0000000..07c6736 --- /dev/null +++ b/tests/enum_to_string/enum.proto @@ -0,0 +1,19 @@ +/* Test enum to string function generation */ + +syntax = "proto2"; + +import "nanopb.proto"; + +option (nanopb_fileopt).enum_to_string = true; + +enum MyEnum { + VALUE1 = 1; + VALUE2 = 2; + VALUE15 = 15; +} + +enum MyShortNameEnum { + option (nanopb_enumopt).long_names = false; + MSNE_VALUE256 = 256; +} + diff --git a/tests/enum_to_string/enum_to_string.c b/tests/enum_to_string/enum_to_string.c new file mode 100644 index 0000000..c4fb31d --- /dev/null +++ b/tests/enum_to_string/enum_to_string.c @@ -0,0 +1,19 @@ +#include <stdio.h> +#include "unittests.h" +#include "enum.pb.h" + +int main() +{ + int status = 0; + TEST(strcmp(MyEnum_name(MyEnum_VALUE1), "VALUE1") == 0); + TEST(strcmp(MyEnum_name(MyEnum_VALUE2), "VALUE2") == 0); + TEST(strcmp(MyEnum_name(MyEnum_VALUE15), "VALUE15") == 0); + TEST(strcmp(MyShortNameEnum_name(MSNE_VALUE256), "MSNE_VALUE256") == 0); + TEST(strcmp(MyShortNameEnum_name(9999), "unknown") == 0); + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} + diff --git a/tests/extensions/SConscript b/tests/extensions/SConscript index 26fc5a3..a2c8742 100644 --- a/tests/extensions/SConscript +++ b/tests/extensions/SConscript @@ -8,8 +8,8 @@ incpath.Append(PROTOCPATH = '$BUILD/alltypes') incpath.Append(CPPPATH = '$BUILD/alltypes') incpath.NanopbProto(["extensions", "extensions.options"]) -enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_encode.o"]) -dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_decode.o"]) +enc = incpath.Program(["encode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) +dec = incpath.Program(["decode_extensions.c", "extensions.pb.c", "$BUILD/alltypes/alltypes.pb$OBJSUFFIX", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(enc) env.RunTest([dec, "encode_extensions.output"]) diff --git a/tests/extensions/extensions.proto b/tests/extensions/extensions.proto index da8432e..fcd5b43 100644 --- a/tests/extensions/extensions.proto +++ b/tests/extensions/extensions.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import 'alltypes.proto'; extend AllTypes { @@ -7,8 +9,8 @@ extend AllTypes { message ExtensionMessage { extend AllTypes { optional ExtensionMessage AllTypes_extensionfield2 = 254; - required ExtensionMessage AllTypes_extensionfield3 = 253; - repeated ExtensionMessage AllTypes_extensionfield4 = 252; + // required ExtensionMessage AllTypes_extensionfield3 = 253; // No longer allowed by protobuf 3 + repeated ExtensionMessage AllTypes_extensionfield4 = 252; } required string test1 = 1; diff --git a/tests/field_size_16/SConscript b/tests/field_size_16/SConscript index 8fee004..99bc344 100644 --- a/tests/field_size_16/SConscript +++ b/tests/field_size_16/SConscript @@ -19,10 +19,14 @@ strict = opts.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode_fields16.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode_fields16.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_fields16.o", "$NANOPB/pb_common.c") # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields16.o"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields16.o"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields16.o", "pb_common_fields16.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields16.o", "pb_common_fields16.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) + +env.RunTest("optionals.output", enc, ARGS = ['1']) +env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) diff --git a/tests/field_size_16/alltypes.options b/tests/field_size_16/alltypes.options index b31e3cf..78dd08d 100644 --- a/tests/field_size_16/alltypes.options +++ b/tests/field_size_16/alltypes.options @@ -1,3 +1,4 @@ * max_size:16 * max_count:5 +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/field_size_16/alltypes.proto b/tests/field_size_16/alltypes.proto index 7494853..4e27059 100644 --- a/tests/field_size_16/alltypes.proto +++ b/tests/field_size_16/alltypes.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message SubMessage { required string substuff1 = 1 [default = "1"]; required int32 substuff2 = 2 [default = 2]; @@ -55,7 +57,7 @@ message AllTypes { required SubMessage req_submsg = 16; required MyEnum req_enum = 17; required EmptyMessage req_emptymsg = 18; - + required bytes req_fbytes = 19; repeated int32 rep_int32 = 21; repeated int64 rep_int64 = 22; @@ -78,6 +80,7 @@ message AllTypes { repeated SubMessage rep_submsg = 10036; repeated MyEnum rep_enum = 10037; repeated EmptyMessage rep_emptymsg = 10038; + repeated bytes rep_fbytes = 10039; optional int32 opt_int32 = 10041 [default = 4041]; optional int64 opt_int64 = 10042 [default = 4042]; @@ -96,10 +99,17 @@ message AllTypes { optional double opt_double = 10053 [default = 4053]; optional string opt_string = 10054 [default = "4054"]; - optional bytes opt_bytes = 10055 [default = "4055"]; + optional bytes opt_bytes = 10055 [default = "\x34\x5C\x00\xff"]; optional SubMessage opt_submsg = 10056; optional MyEnum opt_enum = 10057 [default = Second]; optional EmptyMessage opt_emptymsg = 10058; + optional bytes opt_fbytes = 10059 [default = "4059"]; + + oneof oneof + { + SubMessage oneof_msg1 = 10060; + EmptyMessage oneof_msg2 = 10061; + } // Check that extreme integer values are handled correctly required Limits req_limits = 98; @@ -107,5 +117,7 @@ message AllTypes { // Just to make sure that the size of the fields has been calculated // properly, i.e. otherwise a bug in last field might not be detected. required int32 end = 10099; + + extensions 200 to 255; } diff --git a/tests/field_size_16_proto3/SConscript b/tests/field_size_16_proto3/SConscript new file mode 100644 index 0000000..4a8e16d --- /dev/null +++ b/tests/field_size_16_proto3/SConscript @@ -0,0 +1,34 @@ +# Version of AllTypes test case for protobuf 3 file format. + +Import("env") + +import re +match = None +if 'PROTOC_VERSION' in env: + match = re.search('([0-9]+).([0-9]+).([0-9]+)', env['PROTOC_VERSION']) + +if match: + version = list(map(int, match.groups())) + +# proto3 syntax is supported by protoc >= 3.0.0 +if env.GetOption('clean') or (match and version[0] >= 3): + + env.NanopbProto(["alltypes", "alltypes.options"]) + + # Define the compilation options + opts = env.Clone() + opts.Append(CPPDEFINES = {'PB_FIELD_16BIT': 1}) + + # Build new version of core + strict = opts.Clone() + strict.Append(CFLAGS = strict['CORECFLAGS']) + strict.Object("pb_decode_fields16.o", "$NANOPB/pb_decode.c") + strict.Object("pb_encode_fields16.o", "$NANOPB/pb_encode.c") + strict.Object("pb_common_fields16.o", "$NANOPB/pb_common.c") + + # Now build and run the test normally. + enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields16.o", "pb_common_fields16.o"]) + dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields16.o", "pb_common_fields16.o"]) + + env.RunTest(enc) + env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/field_size_16_proto3/alltypes.options b/tests/field_size_16_proto3/alltypes.options new file mode 100644 index 0000000..edfbe78 --- /dev/null +++ b/tests/field_size_16_proto3/alltypes.options @@ -0,0 +1,4 @@ +* max_size:16 +* max_count:5 +*.*fbytes fixed_length:true max_size:4 +SubMessage.substuff1 max_size:256 diff --git a/tests/field_size_16_proto3/alltypes.proto b/tests/field_size_16_proto3/alltypes.proto new file mode 100644 index 0000000..f66109e --- /dev/null +++ b/tests/field_size_16_proto3/alltypes.proto @@ -0,0 +1,100 @@ +syntax = "proto3"; +// package name placeholder + +message SubMessage { + string substuff1 = 1; + int32 substuff2 = 2; + fixed32 substuff3 = 3; +} + +message EmptyMessage { + +} + +enum HugeEnum { + HE_Zero = 0; + Negative = -2147483647; /* protoc doesn't accept -2147483648 here */ + Positive = 2147483647; +} + +message Limits { + int32 int32_min = 1; + int32 int32_max = 2; + uint32 uint32_min = 3; + uint32 uint32_max = 4; + int64 int64_min = 5; + int64 int64_max = 6; + uint64 uint64_min = 7; + uint64 uint64_max = 8; + HugeEnum enum_min = 9; + HugeEnum enum_max = 10; +} + +enum MyEnum { + Zero = 0; + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + int32 sng_int32 = 1; + int64 sng_int64 = 2; + uint32 sng_uint32 = 3; + uint64 sng_uint64 = 4; + sint32 sng_sint32 = 5; + sint64 sng_sint64 = 6; + bool sng_bool = 7; + + fixed32 sng_fixed32 = 8; + sfixed32 sng_sfixed32= 9; + float sng_float = 10; + + fixed64 sng_fixed64 = 11; + sfixed64 sng_sfixed64= 12; + double sng_double = 13; + + string sng_string = 14; + bytes sng_bytes = 15; + SubMessage sng_submsg = 16; + MyEnum sng_enum = 17; + EmptyMessage sng_emptymsg = 18; + bytes sng_fbytes = 19; + + repeated int32 rep_int32 = 21 [packed = true]; + repeated int64 rep_int64 = 22 [packed = true]; + repeated uint32 rep_uint32 = 23 [packed = true]; + repeated uint64 rep_uint64 = 24 [packed = true]; + repeated sint32 rep_sint32 = 25 [packed = true]; + repeated sint64 rep_sint64 = 26 [packed = true]; + repeated bool rep_bool = 27 [packed = true]; + + repeated fixed32 rep_fixed32 = 28 [packed = true]; + repeated sfixed32 rep_sfixed32= 29 [packed = true]; + repeated float rep_float = 30 [packed = true]; + + repeated fixed64 rep_fixed64 = 31 [packed = true]; + repeated sfixed64 rep_sfixed64= 32 [packed = true]; + repeated double rep_double = 33 [packed = true]; + + repeated string rep_string = 34; + repeated bytes rep_bytes = 35; + repeated SubMessage rep_submsg = 36; + repeated MyEnum rep_enum = 37 [packed = true]; + repeated EmptyMessage rep_emptymsg = 38; + repeated bytes rep_fbytes = 39; + + oneof oneof + { + SubMessage oneof_msg1 = 59; + EmptyMessage oneof_msg2 = 60; + } + + // Check that extreme integer values are handled correctly + Limits req_limits = 98; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + int32 end = 99; +} + diff --git a/tests/field_size_16_proto3/decode_alltypes.c b/tests/field_size_16_proto3/decode_alltypes.c new file mode 100644 index 0000000..6611f8c --- /dev/null +++ b/tests/field_size_16_proto3/decode_alltypes.c @@ -0,0 +1,167 @@ +/* Tests the decoding of all types. + * This is the counterpart of test_encode3. + * Run e.g. ./test_encode3 | ./test_decode3 + */ + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <pb_decode.h> +#include "alltypes.pb.h" +#include "test_helpers.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return false; \ + } + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + AllTypes alltypes = AllTypes_init_zero; + + /* Fill with garbage to better detect initialization errors */ + memset(&alltypes, 0xAA, sizeof(alltypes)); + + if (!pb_decode(stream, AllTypes_fields, &alltypes)) + return false; + + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_int64_count == 5 && alltypes.rep_int64[4] == -2002 && alltypes.rep_int64[0] == 0); + TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); + TEST(alltypes.rep_uint64_count == 5 && alltypes.rep_uint64[4] == 2004 && alltypes.rep_uint64[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_sint64_count == 5 && alltypes.rep_sint64[4] == -2006 && alltypes.rep_sint64[0] == 0); + TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); + + TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); + + TEST(alltypes.rep_fixed64_count == 5 && alltypes.rep_fixed64[4] == 2011 && alltypes.rep_fixed64[0] == 0); + TEST(alltypes.rep_sfixed64_count == 5 && alltypes.rep_sfixed64[4] == -2012 && alltypes.rep_sfixed64[0] == 0); + TEST(alltypes.rep_double_count == 5 && alltypes.rep_double[4] == 2013.0 && alltypes.rep_double[0] == 0.0); + + TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); + TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + + TEST(alltypes.rep_submsg_count == 5); + TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); + TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 0); + + TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + TEST(alltypes.rep_emptymsg_count == 5); + + TEST(alltypes.rep_fbytes_count == 5); + TEST(alltypes.rep_fbytes[0][0] == 0 && alltypes.rep_fbytes[0][3] == 0); + TEST(memcmp(alltypes.rep_fbytes[4], "2019", 4) == 0); + + if (mode == 0) + { + /* Expect default values */ + TEST(alltypes.sng_int32 == 0); + TEST(alltypes.sng_int64 == 0); + TEST(alltypes.sng_uint32 == 0); + TEST(alltypes.sng_uint64 == 0); + TEST(alltypes.sng_sint32 == 0); + TEST(alltypes.sng_sint64 == 0); + TEST(alltypes.sng_bool == false); + + TEST(alltypes.sng_fixed32 == 0); + TEST(alltypes.sng_sfixed32 == 0); + TEST(alltypes.sng_float == 0.0f); + + TEST(alltypes.sng_fixed64 == 0); + TEST(alltypes.sng_sfixed64 == 0); + TEST(alltypes.sng_double == 0.0); + + TEST(strcmp(alltypes.sng_string, "") == 0); + TEST(alltypes.sng_bytes.size == 0); + TEST(strcmp(alltypes.sng_submsg.substuff1, "") == 0); + TEST(alltypes.sng_submsg.substuff2 == 0); + TEST(alltypes.sng_submsg.substuff3 == 0); + TEST(alltypes.sng_enum == MyEnum_Zero); + TEST(alltypes.sng_fbytes[0] == 0 && + alltypes.sng_fbytes[1] == 0 && + alltypes.sng_fbytes[2] == 0 && + alltypes.sng_fbytes[3] == 0); + + TEST(alltypes.which_oneof == 0); + } + else + { + /* Expect filled-in values */ + TEST(alltypes.sng_int32 == 3041); + TEST(alltypes.sng_int64 == 3042); + TEST(alltypes.sng_uint32 == 3043); + TEST(alltypes.sng_uint64 == 3044); + TEST(alltypes.sng_sint32 == 3045); + TEST(alltypes.sng_sint64 == 3046); + TEST(alltypes.sng_bool == true); + + TEST(alltypes.sng_fixed32 == 3048); + TEST(alltypes.sng_sfixed32 == 3049); + TEST(alltypes.sng_float == 3050.0f); + + TEST(alltypes.sng_fixed64 == 3051); + TEST(alltypes.sng_sfixed64 == 3052); + TEST(alltypes.sng_double == 3053.0); + + TEST(strcmp(alltypes.sng_string, "3054") == 0); + TEST(alltypes.sng_bytes.size == 4); + TEST(memcmp(alltypes.sng_bytes.bytes, "3055", 4) == 0); + TEST(strcmp(alltypes.sng_submsg.substuff1, "3056") == 0); + TEST(alltypes.sng_submsg.substuff2 == 3056); + TEST(alltypes.sng_submsg.substuff3 == 0); + TEST(alltypes.sng_enum == MyEnum_Truth); + TEST(memcmp(alltypes.sng_fbytes, "3059", 4) == 0); + + TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); + TEST(strcmp(alltypes.oneof.oneof_msg1.substuff1, "4059") == 0); + TEST(alltypes.oneof.oneof_msg1.substuff2 == 4059); + } + + TEST(alltypes.req_limits.int32_min == INT32_MIN); + TEST(alltypes.req_limits.int32_max == INT32_MAX); + TEST(alltypes.req_limits.uint32_min == 0); + TEST(alltypes.req_limits.uint32_max == UINT32_MAX); + TEST(alltypes.req_limits.int64_min == INT64_MIN); + TEST(alltypes.req_limits.int64_max == INT64_MAX); + TEST(alltypes.req_limits.uint64_min == 0); + TEST(alltypes.req_limits.uint64_max == UINT64_MAX); + TEST(alltypes.req_limits.enum_min == HugeEnum_Negative); + TEST(alltypes.req_limits.enum_max == HugeEnum_Positive); + + TEST(alltypes.end == 1099); + + return true; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[2048]; + size_t count; + pb_istream_t stream; + + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/field_size_16_proto3/encode_alltypes.c b/tests/field_size_16_proto3/encode_alltypes.c new file mode 100644 index 0000000..1da0668 --- /dev/null +++ b/tests/field_size_16_proto3/encode_alltypes.c @@ -0,0 +1,111 @@ +/* Attempts to test all the datatypes supported by ProtoBuf3. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pb_encode.h> +#include "alltypes.pb.h" +#include "test_helpers.h" + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Initialize the structure with constants */ + AllTypes alltypes = AllTypes_init_zero; + + alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; + alltypes.rep_int64_count = 5; alltypes.rep_int64[4] = -2002; + alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; + alltypes.rep_uint64_count = 5; alltypes.rep_uint64[4] = 2004; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; + alltypes.rep_sint64_count = 5; alltypes.rep_sint64[4] = -2006; + alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; + + alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; + alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; + + alltypes.rep_fixed64_count = 5; alltypes.rep_fixed64[4] = 2011; + alltypes.rep_sfixed64_count = 5; alltypes.rep_sfixed64[4] = -2012; + alltypes.rep_double_count = 5; alltypes.rep_double[4] = 2013.0; + + alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); + alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; + memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); + + alltypes.rep_submsg_count = 5; + strcpy(alltypes.rep_submsg[4].substuff1, "2016"); + alltypes.rep_submsg[4].substuff2 = 2016; + alltypes.rep_submsg[4].substuff3 = 2016; + + alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + alltypes.rep_emptymsg_count = 5; + + alltypes.rep_fbytes_count = 5; + memcpy(alltypes.rep_fbytes[4], "2019", 4); + + alltypes.req_limits.int32_min = INT32_MIN; + alltypes.req_limits.int32_max = INT32_MAX; + alltypes.req_limits.uint32_min = 0; + alltypes.req_limits.uint32_max = UINT32_MAX; + alltypes.req_limits.int64_min = INT64_MIN; + alltypes.req_limits.int64_max = INT64_MAX; + alltypes.req_limits.uint64_min = 0; + alltypes.req_limits.uint64_max = UINT64_MAX; + alltypes.req_limits.enum_min = HugeEnum_Negative; + alltypes.req_limits.enum_max = HugeEnum_Positive; + + if (mode != 0) + { + /* Fill in values for singular fields */ + alltypes.sng_int32 = 3041; + alltypes.sng_int64 = 3042; + alltypes.sng_uint32 = 3043; + alltypes.sng_uint64 = 3044; + alltypes.sng_sint32 = 3045; + alltypes.sng_sint64 = 3046; + alltypes.sng_bool = true; + + alltypes.sng_fixed32 = 3048; + alltypes.sng_sfixed32 = 3049; + alltypes.sng_float = 3050.0f; + + alltypes.sng_fixed64 = 3051; + alltypes.sng_sfixed64 = 3052; + alltypes.sng_double = 3053.0; + + strcpy(alltypes.sng_string, "3054"); + alltypes.sng_bytes.size = 4; + memcpy(alltypes.sng_bytes.bytes, "3055", 4); + strcpy(alltypes.sng_submsg.substuff1, "3056"); + alltypes.sng_submsg.substuff2 = 3056; + alltypes.sng_enum = MyEnum_Truth; + memcpy(alltypes.sng_fbytes, "3059", 4); + + alltypes.which_oneof = AllTypes_oneof_msg1_tag; + strcpy(alltypes.oneof.oneof_msg1.substuff1, "4059"); + alltypes.oneof.oneof_msg1.substuff2 = 4059; + } + + alltypes.end = 1099; + + { + uint8_t buffer[AllTypes_size]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } + } +} diff --git a/tests/field_size_32/SConscript b/tests/field_size_32/SConscript index 2a64c6c..650c626 100644 --- a/tests/field_size_32/SConscript +++ b/tests/field_size_32/SConscript @@ -19,10 +19,14 @@ strict = opts.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode_fields32.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode_fields32.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_fields32.o", "$NANOPB/pb_common.c") # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields32.o"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields32.o"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_fields32.o", "pb_common_fields32.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_fields32.o", "pb_common_fields32.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) + +env.RunTest("optionals.output", enc, ARGS = ['1']) +env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) diff --git a/tests/field_size_32/alltypes.options b/tests/field_size_32/alltypes.options index b31e3cf..0d5ab12 100644 --- a/tests/field_size_32/alltypes.options +++ b/tests/field_size_32/alltypes.options @@ -1,3 +1,3 @@ * max_size:16 * max_count:5 - +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/field_size_32/alltypes.proto b/tests/field_size_32/alltypes.proto index 17f17ee..a05e3b9 100644 --- a/tests/field_size_32/alltypes.proto +++ b/tests/field_size_32/alltypes.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message SubMessage { required string substuff1 = 1 [default = "1"]; required int32 substuff2 = 2 [default = 2]; @@ -55,7 +57,7 @@ message AllTypes { required SubMessage req_submsg = 16; required MyEnum req_enum = 17; required EmptyMessage req_emptymsg = 18; - + required bytes req_fbytes = 19; repeated int32 rep_int32 = 21; repeated int64 rep_int64 = 22; @@ -78,6 +80,7 @@ message AllTypes { repeated SubMessage rep_submsg = 10036; repeated MyEnum rep_enum = 10037; repeated EmptyMessage rep_emptymsg = 10038; + repeated bytes rep_fbytes = 10039; optional int32 opt_int32 = 10041 [default = 4041]; optional int64 opt_int64 = 10042 [default = 4042]; @@ -96,10 +99,17 @@ message AllTypes { optional double opt_double = 10053 [default = 4053]; optional string opt_string = 10054 [default = "4054"]; - optional bytes opt_bytes = 10055 [default = "4055"]; + optional bytes opt_bytes = 10055 [default = "\x34\x5C\x00\xff"]; optional SubMessage opt_submsg = 10056; optional MyEnum opt_enum = 10057 [default = Second]; optional EmptyMessage opt_emptymsg = 10058; + optional bytes opt_fbytes = 10059 [default = "4059"]; + + oneof oneof + { + SubMessage oneof_msg1 = 10060; + EmptyMessage oneof_msg2 = 10061; + } // Check that extreme integer values are handled correctly required Limits req_limits = 98; @@ -107,5 +117,7 @@ message AllTypes { // Just to make sure that the size of the fields has been calculated // properly, i.e. otherwise a bug in last field might not be detected. required int32 end = 13432099; + + extensions 200 to 255; } diff --git a/tests/fixed_count/SConscript b/tests/fixed_count/SConscript new file mode 100644 index 0000000..3cecb12 --- /dev/null +++ b/tests/fixed_count/SConscript @@ -0,0 +1,14 @@ +# Test that fixed count option works. + +Import("env") + +env.NanopbProto("fixed_count") +env.Object("fixed_count.pb.c") + +p = env.Program(["fixed_count_unittests.c", + "fixed_count.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_decode.o", + "$COMMON/pb_common.o"]) + +env.RunTest(p) diff --git a/tests/fixed_count/fixed_count.proto b/tests/fixed_count/fixed_count.proto new file mode 100644 index 0000000..e96d1ec --- /dev/null +++ b/tests/fixed_count/fixed_count.proto @@ -0,0 +1,21 @@ +/* Test nanopb fixed count option. */ + +syntax = "proto2"; + +import "nanopb.proto"; + +message Message1 +{ + repeated int32 data = 1 [(nanopb).max_count = 3, (nanopb).fixed_count = true]; +} + +message Message2 +{ + repeated Message1 data = 1 [(nanopb).max_count = 2, (nanopb).fixed_count = true]; +} + +message Message3 +{ + repeated Message2 data1 = 1 [(nanopb).max_count = 2, (nanopb).fixed_count = true]; + repeated Message2 data2 = 2 [(nanopb).max_count = 2, (nanopb).fixed_count = true]; +} diff --git a/tests/fixed_count/fixed_count_unittests.c b/tests/fixed_count/fixed_count_unittests.c new file mode 100644 index 0000000..02bacac --- /dev/null +++ b/tests/fixed_count/fixed_count_unittests.c @@ -0,0 +1,116 @@ +#include <stdio.h> +#include <string.h> +#include <pb_decode.h> +#include <pb_encode.h> +#include "unittests.h" +#include "fixed_count.pb.h" + +int main() +{ + int status = 0; + COMMENT("Test encoding and decoding of repeated fixed count fields"); + + { + pb_byte_t buffer[Message1_size]; + Message1 msg_a = Message1_init_zero; + Message1 msg_b = Message1_init_zero; + + pb_ostream_t ostream; + pb_istream_t istream; + size_t message_length; + + msg_a.data[0] = 1; + msg_a.data[0] = 2; + msg_a.data[0] = 3; + + ostream = pb_ostream_from_buffer(buffer, Message1_size); + TEST(pb_encode(&ostream, Message1_fields, &msg_a)); + message_length = ostream.bytes_written; + + istream = pb_istream_from_buffer(buffer, message_length); + TEST(pb_decode(&istream, Message1_fields, &msg_b)); + + TEST(istream.bytes_left == 0); + TEST(memcmp(&msg_b, &msg_a, sizeof(msg_a)) == 0); + } + + { + pb_byte_t buffer[Message2_size]; + Message2 msg_a = Message2_init_zero; + Message2 msg_b = Message2_init_zero; + + pb_ostream_t ostream; + pb_istream_t istream; + size_t message_length; + + msg_a.data[0].data[0] = 1; + msg_a.data[0].data[1] = 2; + msg_a.data[0].data[2] = 3; + msg_a.data[1].data[0] = 4; + msg_a.data[1].data[1] = 5; + msg_a.data[1].data[2] = 6; + + ostream = pb_ostream_from_buffer(buffer, Message2_size); + TEST(pb_encode(&ostream, Message2_fields, &msg_a)); + message_length = ostream.bytes_written; + + istream = pb_istream_from_buffer(buffer, message_length); + TEST(pb_decode(&istream, Message2_fields, &msg_b)); + + TEST(istream.bytes_left == 0); + TEST(memcmp(&msg_b, &msg_a, sizeof(msg_a)) == 0); + } + + { + pb_byte_t buffer[Message3_size]; + Message3 msg_a = Message3_init_zero; + Message3 msg_b = Message3_init_zero; + + pb_ostream_t ostream; + pb_istream_t istream; + size_t message_length; + + msg_a.data1[0].data[0].data[0] = 1; + msg_a.data1[0].data[0].data[1] = 2; + msg_a.data1[0].data[0].data[2] = 3; + msg_a.data1[0].data[1].data[0] = 4; + msg_a.data1[0].data[1].data[1] = 5; + msg_a.data1[0].data[1].data[2] = 6; + + msg_a.data1[1].data[0].data[0] = 7; + msg_a.data1[1].data[0].data[1] = 8; + msg_a.data1[1].data[0].data[2] = 9; + msg_a.data1[1].data[1].data[0] = 10; + msg_a.data1[1].data[1].data[1] = 11; + msg_a.data1[1].data[1].data[2] = 12; + + msg_a.data2[0].data[0].data[0] = 11; + msg_a.data2[0].data[0].data[1] = 12; + msg_a.data2[0].data[0].data[2] = 13; + msg_a.data2[0].data[1].data[0] = 14; + msg_a.data2[0].data[1].data[1] = 15; + msg_a.data2[0].data[1].data[2] = 16; + + msg_a.data2[1].data[0].data[0] = 17; + msg_a.data2[1].data[0].data[1] = 18; + msg_a.data2[1].data[0].data[2] = 19; + msg_a.data2[1].data[1].data[0] = 110; + msg_a.data2[1].data[1].data[1] = 111; + msg_a.data2[1].data[1].data[2] = 112; + + ostream = pb_ostream_from_buffer(buffer, Message3_size); + TEST(pb_encode(&ostream, Message3_fields, &msg_a)); + message_length = ostream.bytes_written; + + istream = pb_istream_from_buffer(buffer, message_length); + TEST(pb_decode(&istream, Message3_fields, &msg_b)); + + TEST(istream.bytes_left == 0); + TEST(memcmp(&msg_b, &msg_a, sizeof(msg_a)) == 0); + } + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} diff --git a/tests/fuzztest/SConscript b/tests/fuzztest/SConscript new file mode 100644 index 0000000..d2fb689 --- /dev/null +++ b/tests/fuzztest/SConscript @@ -0,0 +1,43 @@ +# Run a fuzz test to verify robustness against corrupted/malicious data. + +Import("env", "malloc_env") + +def set_pkgname(src, dst, pkgname): + data = open(str(src)).read() + placeholder = '// package name placeholder' + assert placeholder in data + data = data.replace(placeholder, 'package %s;' % pkgname) + open(str(dst), 'w').write(data) + +# We want both pointer and static versions of the AllTypes message +# Prefix them with package name. +env.Command("alltypes_static.proto", "#alltypes/alltypes.proto", + lambda target, source, env: set_pkgname(source[0], target[0], 'alltypes_static')) +env.Command("alltypes_pointer.proto", "#alltypes/alltypes.proto", + lambda target, source, env: set_pkgname(source[0], target[0], 'alltypes_pointer')) + +p1 = env.NanopbProto(["alltypes_pointer", "alltypes_pointer.options"]) +p2 = env.NanopbProto(["alltypes_static", "alltypes_static.options"]) +fuzz = malloc_env.Program(["fuzztest.c", + "alltypes_pointer.pb.c", + "alltypes_static.pb.c", + "$COMMON/pb_encode_with_malloc.o", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) + +env.RunTest(fuzz) + +fuzzstub = malloc_env.Program(["fuzzstub.c", + "alltypes_pointer.pb.c", + "alltypes_static.pb.c", + "$COMMON/pb_encode_with_malloc.o", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) + +generate_message = malloc_env.Program(["generate_message.c", + "alltypes_static.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) + diff --git a/tests/fuzztest/alltypes_pointer.options b/tests/fuzztest/alltypes_pointer.options new file mode 100644 index 0000000..7e3ad1e --- /dev/null +++ b/tests/fuzztest/alltypes_pointer.options @@ -0,0 +1,3 @@ +# Generate all fields as pointers. +* type:FT_POINTER +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/fuzztest/alltypes_static.options b/tests/fuzztest/alltypes_static.options new file mode 100644 index 0000000..e197e1d --- /dev/null +++ b/tests/fuzztest/alltypes_static.options @@ -0,0 +1,4 @@ +* max_size:32 +* max_count:8 +*.extensions type:FT_IGNORE +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/fuzztest/fuzzstub.c b/tests/fuzztest/fuzzstub.c new file mode 100644 index 0000000..ec9e2af --- /dev/null +++ b/tests/fuzztest/fuzzstub.c @@ -0,0 +1,189 @@ +/* Fuzz testing for the nanopb core. + * This can be used with external fuzzers, e.g. radamsa. + * It performs most of the same checks as fuzztest, but does not feature data generation. + */ + +#include <pb_decode.h> +#include <pb_encode.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <time.h> +#include <malloc_wrappers.h> +#include "alltypes_static.pb.h" +#include "alltypes_pointer.pb.h" + +#define BUFSIZE 4096 + +static bool do_static_decode(uint8_t *buffer, size_t msglen, bool assert_success) +{ + pb_istream_t stream; + bool status; + + alltypes_static_AllTypes *msg = malloc_with_check(sizeof(alltypes_static_AllTypes)); + stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg); + + if (!status && assert_success) + { + /* Anything that was successfully encoded, should be decodeable. + * One exception: strings without null terminator are encoded up + * to end of buffer, but refused on decode because the terminator + * would not fit. */ + if (strcmp(stream.errmsg, "string overflow") != 0) + assert(status); + } + + free_with_check(msg); + return status; +} + +static bool do_pointer_decode(uint8_t *buffer, size_t msglen, bool assert_success) +{ + pb_istream_t stream; + bool status; + alltypes_pointer_AllTypes *msg; + + msg = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + memset(msg, 0, sizeof(alltypes_pointer_AllTypes)); + stream = pb_istream_from_buffer(buffer, msglen); + + assert(get_alloc_count() == 0); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg); + + if (assert_success) + assert(status); + + pb_release(alltypes_pointer_AllTypes_fields, msg); + assert(get_alloc_count() == 0); + + free_with_check(msg); + + return status; +} + +/* Do a decode -> encode -> decode -> encode roundtrip */ +static void do_static_roundtrip(uint8_t *buffer, size_t msglen) +{ + bool status; + uint8_t *buf2 = malloc_with_check(BUFSIZE); + uint8_t *buf3 = malloc_with_check(BUFSIZE); + size_t msglen2, msglen3; + alltypes_static_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_static_AllTypes)); + alltypes_static_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_static_AllTypes)); + memset(msg1, 0, sizeof(alltypes_static_AllTypes)); + memset(msg2, 0, sizeof(alltypes_static_AllTypes)); + + { + pb_istream_t stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg1); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE); + status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg1); + assert(status); + msglen2 = stream.bytes_written; + } + + { + pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg2); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE); + status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg2); + assert(status); + msglen3 = stream.bytes_written; + } + + assert(msglen2 == msglen3); + assert(memcmp(buf2, buf3, msglen2) == 0); + + free_with_check(msg1); + free_with_check(msg2); + free_with_check(buf2); + free_with_check(buf3); +} + +/* Do decode -> encode -> decode -> encode roundtrip */ +static void do_pointer_roundtrip(uint8_t *buffer, size_t msglen) +{ + bool status; + uint8_t *buf2 = malloc_with_check(BUFSIZE); + uint8_t *buf3 = malloc_with_check(BUFSIZE); + size_t msglen2, msglen3; + alltypes_pointer_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + alltypes_pointer_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + memset(msg1, 0, sizeof(alltypes_pointer_AllTypes)); + memset(msg2, 0, sizeof(alltypes_pointer_AllTypes)); + + { + pb_istream_t stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg1); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE); + status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg1); + assert(status); + msglen2 = stream.bytes_written; + } + + { + pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg2); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE); + status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg2); + assert(status); + msglen3 = stream.bytes_written; + } + + assert(msglen2 == msglen3); + assert(memcmp(buf2, buf3, msglen2) == 0); + + pb_release(alltypes_pointer_AllTypes_fields, msg1); + pb_release(alltypes_pointer_AllTypes_fields, msg2); + free_with_check(msg1); + free_with_check(msg2); + free_with_check(buf2); + free_with_check(buf3); +} + +static void run_iteration() +{ + uint8_t *buffer = malloc_with_check(BUFSIZE); + size_t msglen; + bool status; + + msglen = fread(buffer, 1, BUFSIZE, stdin); + + status = do_static_decode(buffer, msglen, false); + + if (status) + do_static_roundtrip(buffer, msglen); + + status = do_pointer_decode(buffer, msglen, false); + + if (status) + do_pointer_roundtrip(buffer, msglen); + + free_with_check(buffer); +} + +int main(int argc, char **argv) +{ + run_iteration(); + + return 0; +} + diff --git a/tests/fuzztest/fuzztest.c b/tests/fuzztest/fuzztest.c new file mode 100644 index 0000000..ee851ec --- /dev/null +++ b/tests/fuzztest/fuzztest.c @@ -0,0 +1,432 @@ +/* Fuzz testing for the nanopb core. + * Attempts to verify all the properties defined in the security model document. + */ + +#include <pb_decode.h> +#include <pb_encode.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <time.h> +#include <malloc_wrappers.h> +#include "alltypes_static.pb.h" +#include "alltypes_pointer.pb.h" + +static uint64_t random_seed; + +/* Uses xorshift64 here instead of rand() for both speed and + * reproducibility across platforms. */ +static uint32_t rand_word() +{ + random_seed ^= random_seed >> 12; + random_seed ^= random_seed << 25; + random_seed ^= random_seed >> 27; + return random_seed * 2685821657736338717ULL; +} + +/* Get a random integer in range, with approximately flat distribution. */ +static int rand_int(int min, int max) +{ + return rand_word() % (max + 1 - min) + min; +} + +static bool rand_bool() +{ + return rand_word() & 1; +} + +/* Get a random byte, with skewed distribution. + * Important corner cases like 0xFF, 0x00 and 0xFE occur more + * often than other values. */ +static uint8_t rand_byte() +{ + uint32_t w = rand_word(); + uint8_t b = w & 0xFF; + if (w & 0x100000) + b >>= (w >> 8) & 7; + if (w & 0x200000) + b <<= (w >> 12) & 7; + if (w & 0x400000) + b ^= 0xFF; + return b; +} + +/* Get a random length, with skewed distribution. + * Favors the shorter lengths, but always atleast 1. */ +static size_t rand_len(size_t max) +{ + uint32_t w = rand_word(); + size_t s; + if (w & 0x800000) + w &= 3; + else if (w & 0x400000) + w &= 15; + else if (w & 0x200000) + w &= 255; + + s = (w % max); + if (s == 0) + s = 1; + + return s; +} + +/* Fills a buffer with random data with skewed distribution. */ +static void rand_fill(uint8_t *buf, size_t count) +{ + while (count--) + *buf++ = rand_byte(); +} + +/* Fill with random protobuf-like data */ +static size_t rand_fill_protobuf(uint8_t *buf, size_t min_bytes, size_t max_bytes, int min_tag) +{ + pb_ostream_t stream = pb_ostream_from_buffer(buf, max_bytes); + + while(stream.bytes_written < min_bytes) + { + pb_wire_type_t wt = rand_int(0, 3); + if (wt == 3) wt = 5; /* Gap in values */ + + if (!pb_encode_tag(&stream, wt, rand_int(min_tag, min_tag + 512))) + break; + + if (wt == PB_WT_VARINT) + { + uint64_t value; + rand_fill((uint8_t*)&value, sizeof(value)); + pb_encode_varint(&stream, value); + } + else if (wt == PB_WT_64BIT) + { + uint64_t value; + rand_fill((uint8_t*)&value, sizeof(value)); + pb_encode_fixed64(&stream, &value); + } + else if (wt == PB_WT_32BIT) + { + uint32_t value; + rand_fill((uint8_t*)&value, sizeof(value)); + pb_encode_fixed32(&stream, &value); + } + else if (wt == PB_WT_STRING) + { + size_t len; + uint8_t *buf; + + if (min_bytes > stream.bytes_written) + len = rand_len(min_bytes - stream.bytes_written); + else + len = 0; + + buf = malloc(len); + pb_encode_varint(&stream, len); + rand_fill(buf, len); + pb_write(&stream, buf, len); + free(buf); + } + } + + return stream.bytes_written; +} + +/* Given a buffer of data, mess it up a bit */ +static void rand_mess(uint8_t *buf, size_t count) +{ + int m = rand_int(0, 3); + + if (m == 0) + { + /* Replace random substring */ + int s = rand_int(0, count - 1); + int l = rand_len(count - s); + rand_fill(buf + s, l); + } + else if (m == 1) + { + /* Swap random bytes */ + int a = rand_int(0, count - 1); + int b = rand_int(0, count - 1); + int x = buf[a]; + buf[a] = buf[b]; + buf[b] = x; + } + else if (m == 2) + { + /* Duplicate substring */ + int s = rand_int(0, count - 2); + int l = rand_len((count - s) / 2); + memcpy(buf + s + l, buf + s, l); + } + else if (m == 3) + { + /* Add random protobuf noise */ + int s = rand_int(0, count - 1); + int l = rand_len(count - s); + rand_fill_protobuf(buf + s, l, count - s, 1); + } +} + +/* Some default data to put in the message */ +static const alltypes_static_AllTypes initval = alltypes_static_AllTypes_init_default; + +#define BUFSIZE 4096 + +static bool do_static_encode(uint8_t *buffer, size_t *msglen) +{ + pb_ostream_t stream; + bool status; + + /* Allocate a message and fill it with defaults */ + alltypes_static_AllTypes *msg = malloc_with_check(sizeof(alltypes_static_AllTypes)); + memcpy(msg, &initval, sizeof(initval)); + + /* Apply randomness to the data before encoding */ + while (rand_int(0, 7)) + rand_mess((uint8_t*)msg, sizeof(alltypes_static_AllTypes)); + + stream = pb_ostream_from_buffer(buffer, BUFSIZE); + status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg); + assert(stream.bytes_written <= BUFSIZE); + assert(stream.bytes_written <= alltypes_static_AllTypes_size); + + *msglen = stream.bytes_written; + pb_release(alltypes_static_AllTypes_fields, msg); + free_with_check(msg); + + return status; +} + +/* Append or prepend protobuf noise */ +static void do_protobuf_noise(uint8_t *buffer, size_t *msglen) +{ + int m = rand_int(0, 2); + size_t max_size = BUFSIZE - 32 - *msglen; + if (m == 1) + { + /* Prepend */ + uint8_t *tmp = malloc_with_check(BUFSIZE); + size_t s = rand_fill_protobuf(tmp, rand_len(max_size), BUFSIZE - *msglen, 512); + memmove(buffer + s, buffer, *msglen); + memcpy(buffer, tmp, s); + free_with_check(tmp); + *msglen += s; + } + else if (m == 2) + { + /* Append */ + size_t s = rand_fill_protobuf(buffer + *msglen, rand_len(max_size), BUFSIZE - *msglen, 512); + *msglen += s; + } +} + +static bool do_static_decode(uint8_t *buffer, size_t msglen, bool assert_success) +{ + pb_istream_t stream; + bool status; + + alltypes_static_AllTypes *msg = malloc_with_check(sizeof(alltypes_static_AllTypes)); + rand_fill((uint8_t*)msg, sizeof(alltypes_static_AllTypes)); + stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg); + + if (!status && assert_success) + { + /* Anything that was successfully encoded, should be decodeable. + * One exception: strings without null terminator are encoded up + * to end of buffer, but refused on decode because the terminator + * would not fit. */ + if (strcmp(stream.errmsg, "string overflow") != 0) + assert(status); + } + + free_with_check(msg); + return status; +} + +static bool do_pointer_decode(uint8_t *buffer, size_t msglen, bool assert_success) +{ + pb_istream_t stream; + bool status; + alltypes_pointer_AllTypes *msg; + + msg = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + memset(msg, 0, sizeof(alltypes_pointer_AllTypes)); + stream = pb_istream_from_buffer(buffer, msglen); + + assert(get_alloc_count() == 0); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg); + + if (assert_success) + assert(status); + + pb_release(alltypes_pointer_AllTypes_fields, msg); + assert(get_alloc_count() == 0); + + free_with_check(msg); + + return status; +} + +/* Do a decode -> encode -> decode -> encode roundtrip */ +static void do_static_roundtrip(uint8_t *buffer, size_t msglen) +{ + bool status; + uint8_t *buf2 = malloc_with_check(BUFSIZE); + uint8_t *buf3 = malloc_with_check(BUFSIZE); + size_t msglen2, msglen3; + alltypes_static_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_static_AllTypes)); + alltypes_static_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_static_AllTypes)); + memset(msg1, 0, sizeof(alltypes_static_AllTypes)); + memset(msg2, 0, sizeof(alltypes_static_AllTypes)); + + { + pb_istream_t stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg1); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE); + status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg1); + assert(status); + msglen2 = stream.bytes_written; + } + + { + pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2); + status = pb_decode(&stream, alltypes_static_AllTypes_fields, msg2); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE); + status = pb_encode(&stream, alltypes_static_AllTypes_fields, msg2); + assert(status); + msglen3 = stream.bytes_written; + } + + assert(msglen2 == msglen3); + assert(memcmp(buf2, buf3, msglen2) == 0); + + free_with_check(msg1); + free_with_check(msg2); + free_with_check(buf2); + free_with_check(buf3); +} + +/* Do decode -> encode -> decode -> encode roundtrip */ +static void do_pointer_roundtrip(uint8_t *buffer, size_t msglen) +{ + bool status; + uint8_t *buf2 = malloc_with_check(BUFSIZE); + uint8_t *buf3 = malloc_with_check(BUFSIZE); + size_t msglen2, msglen3; + alltypes_pointer_AllTypes *msg1 = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + alltypes_pointer_AllTypes *msg2 = malloc_with_check(sizeof(alltypes_pointer_AllTypes)); + memset(msg1, 0, sizeof(alltypes_pointer_AllTypes)); + memset(msg2, 0, sizeof(alltypes_pointer_AllTypes)); + + { + pb_istream_t stream = pb_istream_from_buffer(buffer, msglen); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg1); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf2, BUFSIZE); + status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg1); + assert(status); + msglen2 = stream.bytes_written; + } + + { + pb_istream_t stream = pb_istream_from_buffer(buf2, msglen2); + status = pb_decode(&stream, alltypes_pointer_AllTypes_fields, msg2); + assert(status); + } + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf3, BUFSIZE); + status = pb_encode(&stream, alltypes_pointer_AllTypes_fields, msg2); + assert(status); + msglen3 = stream.bytes_written; + } + + assert(msglen2 == msglen3); + assert(memcmp(buf2, buf3, msglen2) == 0); + + pb_release(alltypes_pointer_AllTypes_fields, msg1); + pb_release(alltypes_pointer_AllTypes_fields, msg2); + free_with_check(msg1); + free_with_check(msg2); + free_with_check(buf2); + free_with_check(buf3); +} + +static void run_iteration() +{ + uint8_t *buffer = malloc_with_check(BUFSIZE); + size_t msglen; + bool status; + + rand_fill(buffer, BUFSIZE); + + if (do_static_encode(buffer, &msglen)) + { + do_protobuf_noise(buffer, &msglen); + + status = do_static_decode(buffer, msglen, true); + + if (status) + do_static_roundtrip(buffer, msglen); + + status = do_pointer_decode(buffer, msglen, true); + + if (status) + do_pointer_roundtrip(buffer, msglen); + + /* Apply randomness to the encoded data */ + while (rand_bool()) + rand_mess(buffer, BUFSIZE); + + /* Apply randomness to encoded data length */ + if (rand_bool()) + msglen = rand_int(0, BUFSIZE); + + status = do_static_decode(buffer, msglen, false); + do_pointer_decode(buffer, msglen, status); + + if (status) + { + do_static_roundtrip(buffer, msglen); + do_pointer_roundtrip(buffer, msglen); + } + } + + free_with_check(buffer); +} + +int main(int argc, char **argv) +{ + int i; + if (argc > 1) + { + random_seed = atol(argv[1]); + } + else + { + random_seed = time(NULL); + } + + fprintf(stderr, "Random seed: %llu\n", (long long unsigned)random_seed); + + for (i = 0; i < 10000; i++) + { + run_iteration(); + } + + return 0; +} + diff --git a/tests/fuzztest/generate_message.c b/tests/fuzztest/generate_message.c new file mode 100644 index 0000000..6e49299 --- /dev/null +++ b/tests/fuzztest/generate_message.c @@ -0,0 +1,101 @@ +/* Generates a random, valid protobuf message. Useful to seed + * external fuzzers such as afl-fuzz. + */ + +#include <pb_encode.h> +#include <pb_common.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <assert.h> +#include <time.h> +#include "alltypes_static.pb.h" + +static uint64_t random_seed; + +/* Uses xorshift64 here instead of rand() for both speed and + * reproducibility across platforms. */ +static uint32_t rand_word() +{ + random_seed ^= random_seed >> 12; + random_seed ^= random_seed << 25; + random_seed ^= random_seed >> 27; + return random_seed * 2685821657736338717ULL; +} + +/* Fills a buffer with random data. */ +static void rand_fill(uint8_t *buf, size_t count) +{ + while (count--) + { + *buf++ = rand_word() & 0xff; + } +} + +/* Check that size/count fields do not exceed their max size. + * Otherwise we would have to loop pretty long in generate_message(). + * Note that there may still be a few encoding errors from submessages. + */ +static void limit_sizes(alltypes_static_AllTypes *msg) +{ + pb_field_iter_t iter; + pb_field_iter_begin(&iter, alltypes_static_AllTypes_fields, msg); + while (pb_field_iter_next(&iter)) + { + if (PB_LTYPE(iter.pos->type) == PB_LTYPE_BYTES) + { + ((pb_bytes_array_t*)iter.pData)->size %= iter.pos->data_size - PB_BYTES_ARRAY_T_ALLOCSIZE(0); + } + + if (PB_HTYPE(iter.pos->type) == PB_HTYPE_REPEATED) + { + *((pb_size_t*)iter.pSize) %= iter.pos->array_size; + } + + if (PB_HTYPE(iter.pos->type) == PB_HTYPE_ONEOF) + { + /* Set the oneof to this message type with 50% chance. */ + if (rand_word() & 1) + { + *((pb_size_t*)iter.pSize) = iter.pos->tag; + } + } + } +} + +static void generate_message() +{ + alltypes_static_AllTypes msg; + uint8_t buf[8192]; + pb_ostream_t stream = {0}; + + do { + if (stream.errmsg) + fprintf(stderr, "Encoder error: %s\n", stream.errmsg); + + stream = pb_ostream_from_buffer(buf, sizeof(buf)); + rand_fill((void*)&msg, sizeof(msg)); + limit_sizes(&msg); + } while (!pb_encode(&stream, alltypes_static_AllTypes_fields, &msg)); + + fwrite(buf, 1, stream.bytes_written, stdout); +} + +int main(int argc, char **argv) +{ + if (argc > 1) + { + random_seed = atol(argv[1]); + } + else + { + random_seed = time(NULL); + } + + fprintf(stderr, "Random seed: %llu\n", (long long unsigned)random_seed); + + generate_message(); + + return 0; +} + diff --git a/tests/fuzztest/run_radamsa.sh b/tests/fuzztest/run_radamsa.sh new file mode 100755 index 0000000..52cd40a --- /dev/null +++ b/tests/fuzztest/run_radamsa.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +TMP=`tempfile` + +echo $TMP +while true +do + radamsa sample_data/* > $TMP + $1 < $TMP + test $? -gt 127 && break +done + diff --git a/tests/fuzztest/sample_data/sample1.pb b/tests/fuzztest/sample_data/sample1.pb Binary files differnew file mode 100644 index 0000000..0752788 --- /dev/null +++ b/tests/fuzztest/sample_data/sample1.pb diff --git a/tests/fuzztest/sample_data/sample2.pb b/tests/fuzztest/sample_data/sample2.pb Binary files differnew file mode 100644 index 0000000..cc89f91 --- /dev/null +++ b/tests/fuzztest/sample_data/sample2.pb diff --git a/tests/inline/SConscript b/tests/inline/SConscript new file mode 100644 index 0000000..34371fd --- /dev/null +++ b/tests/inline/SConscript @@ -0,0 +1,16 @@ +# Test that inlined bytes fields work. + +Import("env") + +env.NanopbProto("inline") +env.Object("inline.pb.c") + +env.Match(["inline.pb.h", "inline.expected"]) + +p = env.Program(["inline_unittests.c", + "inline.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_decode.o", + "$COMMON/pb_common.o"]) + +env.RunTest(p) diff --git a/tests/inline/inline.expected b/tests/inline/inline.expected new file mode 100644 index 0000000..593e972 --- /dev/null +++ b/tests/inline/inline.expected @@ -0,0 +1,3 @@ +pb_byte_t data\[32\]; +bool has_data; +pb_byte_t data\[64\]; diff --git a/tests/inline/inline.proto b/tests/inline/inline.proto new file mode 100644 index 0000000..6e511f0 --- /dev/null +++ b/tests/inline/inline.proto @@ -0,0 +1,17 @@ +/* Test nanopb option parsing. + * options.expected lists the patterns that are searched for in the output. + */ + +syntax = "proto2"; + +import "nanopb.proto"; + +message Message1 +{ + required bytes data = 1 [(nanopb).type = FT_INLINE, (nanopb).max_size = 32]; +} + +message Message2 +{ + optional bytes data = 1 [(nanopb).type = FT_INLINE, (nanopb).max_size = 64]; +} diff --git a/tests/inline/inline_unittests.c b/tests/inline/inline_unittests.c new file mode 100644 index 0000000..b5834c7 --- /dev/null +++ b/tests/inline/inline_unittests.c @@ -0,0 +1,73 @@ +#include <stdio.h> +#include <string.h> +#include <pb_decode.h> +#include <pb_encode.h> +#include "unittests.h" +#include "inline.pb.h" + +int main() +{ + int status = 0; + int i = 0; + COMMENT("Test inline byte fields"); + + { + Message1 msg1 = Message1_init_zero; + TEST(sizeof(msg1.data) == 32); + } + + { + Message1 msg1 = Message1_init_zero; + pb_byte_t msg1_buffer[Message1_size]; + pb_ostream_t ostream = pb_ostream_from_buffer(msg1_buffer, Message1_size); + Message1 msg1_deserialized = Message1_init_zero; + pb_istream_t istream = pb_istream_from_buffer(msg1_buffer, Message1_size); + + for (i = 0; i < 32; i++) { + msg1.data[i] = i; + } + + TEST(pb_encode(&ostream, Message1_fields, &msg1)); + TEST(ostream.bytes_written == Message1_size); + + TEST(pb_decode(&istream, Message1_fields, &msg1_deserialized)); + + TEST(istream.bytes_left == 0); + TEST(memcmp(&msg1_deserialized, &msg1, sizeof(msg1)) == 0); + } + + { + Message2 msg2 = {true, {0}}; + Message2 msg2_no_data = {false, {1}}; + pb_byte_t msg2_buffer[Message2_size]; + pb_ostream_t ostream = pb_ostream_from_buffer(msg2_buffer, Message2_size); + Message2 msg2_deserialized = Message2_init_zero; + pb_istream_t istream = pb_istream_from_buffer(msg2_buffer, Message2_size); + + for (i = 0; i < 64; i++) { + msg2.data[i] = i; + } + + TEST(pb_encode(&ostream, Message2_fields, &msg2)); + TEST(ostream.bytes_written == Message2_size); + + TEST(pb_decode(&istream, Message2_fields, &msg2_deserialized)); + + TEST(istream.bytes_left == 0); + TEST(memcmp(&msg2_deserialized, &msg2, sizeof(msg2)) == 0); + TEST(msg2_deserialized.has_data); + + memset(msg2_buffer, 0, sizeof(msg2_buffer)); + ostream = pb_ostream_from_buffer(msg2_buffer, Message2_size); + TEST(pb_encode(&ostream, Message2_fields, &msg2_no_data)); + istream = pb_istream_from_buffer(msg2_buffer, Message2_size); + TEST(pb_decode(&istream, Message2_fields, &msg2_deserialized)); + TEST(!msg2_deserialized.has_data); + TEST(memcmp(&msg2_deserialized, &msg2, sizeof(msg2)) != 0); + } + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +} diff --git a/tests/intsizes/SConscript b/tests/intsizes/SConscript new file mode 100644 index 0000000..a90680b --- /dev/null +++ b/tests/intsizes/SConscript @@ -0,0 +1,12 @@ +# Test that the int_size option in .proto works. + +Import('env') + +env.NanopbProto('intsizes') + +p = env.Program(["intsizes_unittests.c", + "intsizes.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_decode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) diff --git a/tests/intsizes/intsizes.proto b/tests/intsizes/intsizes.proto new file mode 100644 index 0000000..91444d4 --- /dev/null +++ b/tests/intsizes/intsizes.proto @@ -0,0 +1,41 @@ +/* Test the integer size overriding in nanopb options. + * This allows to use 8- and 16-bit integer variables, which are not supported + * directly by Google Protobuf. + * + * The int_size setting will override the number of bits, but keep the type + * otherwise. E.g. uint32 + IS_8 => uint8_t + */ + +syntax = "proto2"; + +import 'nanopb.proto'; + +message IntSizes { + required int32 req_int8 = 1 [(nanopb).int_size = IS_8]; + required uint32 req_uint8 = 2 [(nanopb).int_size = IS_8]; + required sint32 req_sint8 = 3 [(nanopb).int_size = IS_8]; + required int32 req_int16 = 4 [(nanopb).int_size = IS_16]; + required uint32 req_uint16 = 5 [(nanopb).int_size = IS_16]; + required sint32 req_sint16 = 6 [(nanopb).int_size = IS_16]; + required int32 req_int32 = 7 [(nanopb).int_size = IS_32]; + required uint32 req_uint32 = 8 [(nanopb).int_size = IS_32]; + required sint32 req_sint32 = 9 [(nanopb).int_size = IS_32]; + required int32 req_int64 = 10 [(nanopb).int_size = IS_64]; + required uint32 req_uint64 = 11 [(nanopb).int_size = IS_64]; + required sint32 req_sint64 = 12 [(nanopb).int_size = IS_64]; +} + +message DefaultSizes { + required int32 req_int8 = 1 ; + required uint32 req_uint8 = 2 ; + required sint32 req_sint8 = 3 ; + required int32 req_int16 = 4 ; + required uint32 req_uint16 = 5 ; + required sint32 req_sint16 = 6 ; + required int32 req_int32 = 7 ; + required uint32 req_uint32 = 8 ; + required sint32 req_sint32 = 9 ; + required int64 req_int64 = 10; + required uint64 req_uint64 = 11; + required sint64 req_sint64 = 12; +} diff --git a/tests/intsizes/intsizes_unittests.c b/tests/intsizes/intsizes_unittests.c new file mode 100644 index 0000000..79ef036 --- /dev/null +++ b/tests/intsizes/intsizes_unittests.c @@ -0,0 +1,122 @@ +#include <stdio.h> +#include <string.h> +#include <pb_decode.h> +#include <pb_encode.h> +#include "unittests.h" +#include "intsizes.pb.h" + +#define S(x) pb_istream_from_buffer((uint8_t*)x, sizeof(x) - 1) + +/* This is a macro instead of function in order to get the actual values + * into the TEST() lines in output */ +#define TEST_ROUNDTRIP(int8, uint8, sint8, \ + int16, uint16, sint16, \ + int32, uint32, sint32, \ + int64, uint64, sint64, expected_result) \ +{ \ + uint8_t buffer1[128], buffer2[128]; \ + size_t msgsize; \ + DefaultSizes msg1 = DefaultSizes_init_zero; \ + IntSizes msg2 = IntSizes_init_zero; \ + \ + msg1.req_int8 = int8; \ + msg1.req_uint8 = uint8; \ + msg1.req_sint8 = sint8; \ + msg1.req_int16 = int16; \ + msg1.req_uint16 = uint16; \ + msg1.req_sint16 = sint16; \ + msg1.req_int32 = int32; \ + msg1.req_uint32 = uint32; \ + msg1.req_sint32 = sint32; \ + msg1.req_int64 = int64; \ + msg1.req_uint64 = uint64; \ + msg1.req_sint64 = sint64; \ + \ + { \ + pb_ostream_t s = pb_ostream_from_buffer(buffer1, sizeof(buffer1)); \ + TEST(pb_encode(&s, DefaultSizes_fields, &msg1)); \ + msgsize = s.bytes_written; \ + } \ + \ + { \ + pb_istream_t s = pb_istream_from_buffer(buffer1, msgsize); \ + TEST(pb_decode(&s, IntSizes_fields, &msg2) == expected_result); \ + if (expected_result) \ + { \ + TEST( (int64_t)msg2.req_int8 == int8); \ + TEST((uint64_t)msg2.req_uint8 == uint8); \ + TEST( (int64_t)msg2.req_sint8 == sint8); \ + TEST( (int64_t)msg2.req_int16 == int16); \ + TEST((uint64_t)msg2.req_uint16 == uint16); \ + TEST( (int64_t)msg2.req_sint16 == sint16); \ + TEST( (int64_t)msg2.req_int32 == int32); \ + TEST((uint64_t)msg2.req_uint32 == uint32); \ + TEST( (int64_t)msg2.req_sint32 == sint32); \ + TEST( (int64_t)msg2.req_int64 == int64); \ + TEST((uint64_t)msg2.req_uint64 == uint64); \ + TEST( (int64_t)msg2.req_sint64 == sint64); \ + } \ + } \ + \ + if (expected_result) \ + { \ + pb_ostream_t s = pb_ostream_from_buffer(buffer2, sizeof(buffer2)); \ + TEST(pb_encode(&s, IntSizes_fields, &msg2)); \ + TEST(s.bytes_written == msgsize); \ + TEST(memcmp(buffer1, buffer2, msgsize) == 0); \ + } \ +} + +int main() +{ + int status = 0; + + { + IntSizes msg = IntSizes_init_zero; + + COMMENT("Test field sizes"); + TEST(sizeof(msg.req_int8) == 1); + TEST(sizeof(msg.req_uint8) == 1); + TEST(sizeof(msg.req_sint8) == 1); + TEST(sizeof(msg.req_int16) == 2); + TEST(sizeof(msg.req_uint16) == 2); + TEST(sizeof(msg.req_sint16) == 2); + TEST(sizeof(msg.req_int32) == 4); + TEST(sizeof(msg.req_uint32) == 4); + TEST(sizeof(msg.req_sint32) == 4); + TEST(sizeof(msg.req_int64) == 8); + TEST(sizeof(msg.req_uint64) == 8); + TEST(sizeof(msg.req_sint64) == 8); + } + + COMMENT("Test roundtrip at maximum value"); + TEST_ROUNDTRIP(127, 255, 127, + 32767, 65535, 32767, + INT32_MAX, UINT32_MAX, INT32_MAX, + INT64_MAX, UINT64_MAX, INT64_MAX, true); + + COMMENT("Test roundtrip at minimum value"); + TEST_ROUNDTRIP(-128, 0, -128, + -32768, 0, -32768, + INT32_MIN, 0, INT32_MIN, + INT64_MIN, 0, INT64_MIN, true); + + COMMENT("Test overflow detection"); + TEST_ROUNDTRIP(-129, 0, -128, + -32768, 0, -32768, + INT32_MIN, 0, INT32_MIN, + INT64_MIN, 0, INT64_MIN, false); + TEST_ROUNDTRIP(127, 256, 127, + 32767, 65535, 32767, + INT32_MAX, UINT32_MAX, INT32_MAX, + INT64_MAX, UINT64_MAX, INT64_MAX, false); + TEST_ROUNDTRIP(-128, 0, -128, + -32768, 0, -32769, + INT32_MIN, 0, INT32_MIN, + INT64_MIN, 0, INT64_MIN, false); + + if (status != 0) + fprintf(stdout, "\n\nSome tests FAILED!\n"); + + return status; +}
\ No newline at end of file diff --git a/tests/io_errors/SConscript b/tests/io_errors/SConscript new file mode 100644 index 0000000..60146cc --- /dev/null +++ b/tests/io_errors/SConscript @@ -0,0 +1,15 @@ +# Simulate io errors when encoding and decoding + +Import("env") + +c = Copy("$TARGET", "$SOURCE") +env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) + +env.NanopbProto(["alltypes", "alltypes.options"]) + +ioerr = env.Program(["io_errors.c", "alltypes.pb.c", + "$COMMON/pb_encode.o", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) + +env.RunTest("io_errors.output", [ioerr, "$BUILD/alltypes/encode_alltypes.output"]) + + diff --git a/tests/io_errors/alltypes.options b/tests/io_errors/alltypes.options new file mode 100644 index 0000000..0d5ab12 --- /dev/null +++ b/tests/io_errors/alltypes.options @@ -0,0 +1,3 @@ +* max_size:16 +* max_count:5 +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/io_errors/io_errors.c b/tests/io_errors/io_errors.c new file mode 100644 index 0000000..76f35b0 --- /dev/null +++ b/tests/io_errors/io_errors.c @@ -0,0 +1,140 @@ +/* Simulate IO errors after each byte in a stream. + * Verifies proper error propagation. + */ + +#include <stdio.h> +#include <pb_decode.h> +#include <pb_encode.h> +#include "alltypes.pb.h" +#include "test_helpers.h" + +typedef struct +{ + uint8_t *buffer; + size_t fail_after; +} faulty_stream_t; + +bool read_callback(pb_istream_t *stream, uint8_t *buf, size_t count) +{ + faulty_stream_t *state = stream->state; + + while (count--) + { + if (state->fail_after == 0) + PB_RETURN_ERROR(stream, "simulated"); + state->fail_after--; + *buf++ = *state->buffer++; + } + + return true; +} +bool write_callback(pb_ostream_t *stream, const uint8_t *buf, size_t count) +{ + faulty_stream_t *state = stream->state; + + while (count--) + { + if (state->fail_after == 0) + PB_RETURN_ERROR(stream, "simulated"); + state->fail_after--; + *state->buffer++ = *buf++; + } + + return true; +} + +int main() +{ + uint8_t buffer[2048]; + size_t msglen; + AllTypes msg = AllTypes_init_zero; + + /* Get some base data to run the tests with */ + SET_BINARY_MODE(stdin); + msglen = fread(buffer, 1, sizeof(buffer), stdin); + + /* Test IO errors on decoding */ + { + bool status; + pb_istream_t stream = {&read_callback, NULL, SIZE_MAX}; + faulty_stream_t fs; + size_t i; + + for (i = 0; i < msglen; i++) + { + stream.bytes_left = msglen; + stream.state = &fs; + fs.buffer = buffer; + fs.fail_after = i; + + status = pb_decode(&stream, AllTypes_fields, &msg); + if (status != false) + { + fprintf(stderr, "Unexpected success in decode\n"); + return 2; + } + else if (strcmp(stream.errmsg, "simulated") != 0) + { + fprintf(stderr, "Wrong error in decode: %s\n", stream.errmsg); + return 3; + } + } + + stream.bytes_left = msglen; + stream.state = &fs; + fs.buffer = buffer; + fs.fail_after = msglen; + status = pb_decode(&stream, AllTypes_fields, &msg); + + if (!status) + { + fprintf(stderr, "Decoding failed: %s\n", stream.errmsg); + return 4; + } + } + + /* Test IO errors on encoding */ + { + bool status; + pb_ostream_t stream = {&write_callback, NULL, SIZE_MAX, 0}; + faulty_stream_t fs; + size_t i; + + for (i = 0; i < msglen; i++) + { + stream.max_size = msglen; + stream.bytes_written = 0; + stream.state = &fs; + fs.buffer = buffer; + fs.fail_after = i; + + status = pb_encode(&stream, AllTypes_fields, &msg); + if (status != false) + { + fprintf(stderr, "Unexpected success in encode\n"); + return 5; + } + else if (strcmp(stream.errmsg, "simulated") != 0) + { + fprintf(stderr, "Wrong error in encode: %s\n", stream.errmsg); + return 6; + } + } + + stream.max_size = msglen; + stream.bytes_written = 0; + stream.state = &fs; + fs.buffer = buffer; + fs.fail_after = msglen; + status = pb_encode(&stream, AllTypes_fields, &msg); + + if (!status) + { + fprintf(stderr, "Encoding failed: %s\n", stream.errmsg); + return 7; + } + } + + return 0; +} + diff --git a/tests/io_errors_pointers/SConscript b/tests/io_errors_pointers/SConscript new file mode 100644 index 0000000..03727df --- /dev/null +++ b/tests/io_errors_pointers/SConscript @@ -0,0 +1,26 @@ +# Simulate io errors when encoding and decoding + +Import("env", "malloc_env") + +c = Copy("$TARGET", "$SOURCE") +env.Command("alltypes.proto", "#alltypes/alltypes.proto", c) +env.Command("io_errors.c", "#io_errors/io_errors.c", c) + +env.NanopbProto(["alltypes", "alltypes.options"]) + +ioerr = env.Program(["io_errors.c", "alltypes.pb.c", + "$COMMON/pb_encode_with_malloc.o", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) + +# Run tests under valgrind if available +valgrind = env.WhereIs('valgrind') +kwargs = {} +if valgrind: + kwargs['COMMAND'] = valgrind + kwargs['ARGS'] = ["-q", "--error-exitcode=99", ioerr[0].abspath] + +env.RunTest("io_errors.output", [ioerr, "$BUILD/alltypes/encode_alltypes.output"], **kwargs) + + diff --git a/tests/io_errors_pointers/alltypes.options b/tests/io_errors_pointers/alltypes.options new file mode 100644 index 0000000..7e3ad1e --- /dev/null +++ b/tests/io_errors_pointers/alltypes.options @@ -0,0 +1,3 @@ +# Generate all fields as pointers. +* type:FT_POINTER +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/map/SConscript b/tests/map/SConscript new file mode 100644 index 0000000..f23e8ee --- /dev/null +++ b/tests/map/SConscript @@ -0,0 +1,21 @@ +# Example / test for handling 'map' type using the backwards compatibility +# in protobuf specification: +# https://developers.google.com/protocol-buffers/docs/proto3#maps + +Import('env') + +env.NanopbProto(['map', 'map.options']) + +enc = env.Program(['encode_map.c', + 'map.pb.c', + '$COMMON/pb_encode.o', + '$COMMON/pb_common.o']) + +dec = env.Program(['decode_map.c', + 'map.pb.c', + '$COMMON/pb_decode.o', + '$COMMON/pb_common.o']) + +env.RunTest("message.pb", enc) +env.RunTest("message.txt", [dec, 'message.pb']) + diff --git a/tests/map/decode_map.c b/tests/map/decode_map.c new file mode 100644 index 0000000..c798b03 --- /dev/null +++ b/tests/map/decode_map.c @@ -0,0 +1,60 @@ +/* Decode a message using map field */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pb_decode.h> +#include "map.pb.h" +#include "test_helpers.h" +#include "unittests.h" + +/* Helper function to find an entry in the list. Not as efficient as a real + * hashmap or similar would be, but suitable for small arrays. */ +MyMessage_NumbersEntry *find_entry(MyMessage *msg, const char *key) +{ + int i; + for (i = 0; i < msg->numbers_count; i++) + { + if (strcmp(msg->numbers[i].key, key) == 0) + { + return &msg->numbers[i]; + } + } + return NULL; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[MyMessage_size]; + size_t count; + + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + if (!feof(stdin)) + { + printf("Message does not fit in buffer\n"); + return 1; + } + + { + int status = 0; + MyMessage msg = MyMessage_init_zero; + MyMessage_NumbersEntry *e; + pb_istream_t stream = pb_istream_from_buffer(buffer, count); + + if (!pb_decode(&stream, MyMessage_fields, &msg)) + { + fprintf(stderr, "Decoding failed\n"); + return 2; + } + + TEST((e = find_entry(&msg, "one")) && e->value == 1); + TEST((e = find_entry(&msg, "two")) && e->value == 2); + TEST((e = find_entry(&msg, "seven")) && e->value == 7); + TEST(!find_entry(&msg, "zero")); + + return status; + } +} + diff --git a/tests/map/encode_map.c b/tests/map/encode_map.c new file mode 100644 index 0000000..bd4ec12 --- /dev/null +++ b/tests/map/encode_map.c @@ -0,0 +1,37 @@ +/* Encode a message using map field */ + +#include <stdio.h> +#include <stdlib.h> +#include <pb_encode.h> +#include "map.pb.h" +#include "test_helpers.h" + +int main(int argc, char **argv) +{ + uint8_t buffer[MyMessage_size]; + MyMessage msg = MyMessage_init_zero; + pb_ostream_t stream; + + /* Fill in the map entries */ + msg.numbers_count = 3; + strncpy(msg.numbers[0].key, "one", sizeof(msg.numbers[0].key)); + strncpy(msg.numbers[1].key, "two", sizeof(msg.numbers[1].key)); + strncpy(msg.numbers[2].key, "seven", sizeof(msg.numbers[2].key)); + msg.numbers[0].value = 1; + msg.numbers[1].value = 2; + msg.numbers[2].value = 7; + + stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + if (pb_encode(&stream, MyMessage_fields, &msg)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } +} diff --git a/tests/map/map.options b/tests/map/map.options new file mode 100644 index 0000000..1d7ebb6 --- /dev/null +++ b/tests/map/map.options @@ -0,0 +1,2 @@ +MyMessage.numbers max_count:10 +MyMessage.NumbersEntry.key max_size:16 diff --git a/tests/map/map.proto b/tests/map/map.proto new file mode 100644 index 0000000..f503625 --- /dev/null +++ b/tests/map/map.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +message MyMessage { + map<string,uint32> numbers = 1; +} + diff --git a/tests/mem_release/SConscript b/tests/mem_release/SConscript new file mode 100644 index 0000000..6754e28 --- /dev/null +++ b/tests/mem_release/SConscript @@ -0,0 +1,13 @@ +Import("env", "malloc_env") + +env.NanopbProto("mem_release.proto") + +test = malloc_env.Program(["mem_release.c", + "mem_release.pb.c", + "$COMMON/pb_encode_with_malloc.o", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) + +env.RunTest(test) + diff --git a/tests/mem_release/mem_release.c b/tests/mem_release/mem_release.c new file mode 100644 index 0000000..dc6f87d --- /dev/null +++ b/tests/mem_release/mem_release.c @@ -0,0 +1,187 @@ +/* Make sure that all fields are freed in various scenarios. */ + +#include <pb_decode.h> +#include <pb_encode.h> +#include <malloc_wrappers.h> +#include <stdio.h> +#include <test_helpers.h> +#include "mem_release.pb.h" + +#define TEST(x) if (!(x)) { \ + fprintf(stderr, "Test " #x " on line %d failed.\n", __LINE__); \ + return false; \ + } + +static char *test_str_arr[] = {"1", "2", ""}; +static SubMessage test_msg_arr[] = {SubMessage_init_zero, SubMessage_init_zero}; +static pb_extension_t ext1, ext2; + +static void fill_TestMessage(TestMessage *msg) +{ + msg->static_req_submsg.dynamic_str = "12345"; + msg->static_req_submsg.dynamic_str_arr_count = 3; + msg->static_req_submsg.dynamic_str_arr = test_str_arr; + msg->static_req_submsg.dynamic_submsg_count = 2; + msg->static_req_submsg.dynamic_submsg = test_msg_arr; + msg->static_req_submsg.dynamic_submsg[1].dynamic_str = "abc"; + msg->static_opt_submsg.dynamic_str = "abc"; + msg->static_rep_submsg_count = 2; + msg->static_rep_submsg[1].dynamic_str = "abc"; + msg->has_static_opt_submsg = true; + msg->dynamic_submsg = &msg->static_req_submsg; + + msg->extensions = &ext1; + ext1.type = &dynamic_ext; + ext1.dest = &msg->static_req_submsg; + ext1.next = &ext2; + ext2.type = &static_ext; + ext2.dest = &msg->static_req_submsg; + ext2.next = NULL; +} + +/* Basic fields, nested submessages, extensions */ +static bool test_TestMessage() +{ + uint8_t buffer[256]; + size_t msgsize; + + /* Construct a message with various fields filled in */ + { + TestMessage msg = TestMessage_init_zero; + pb_ostream_t stream; + + fill_TestMessage(&msg); + + stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + if (!pb_encode(&stream, TestMessage_fields, &msg)) + { + fprintf(stderr, "Encode failed: %s\n", PB_GET_ERROR(&stream)); + return false; + } + msgsize = stream.bytes_written; + } + + /* Output encoded message for debug */ + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, msgsize, stdout); + + /* Decode memory using dynamic allocation */ + { + TestMessage msg = TestMessage_init_zero; + pb_istream_t stream; + SubMessage ext2_dest; + + msg.extensions = &ext1; + ext1.type = &dynamic_ext; + ext1.dest = NULL; + ext1.next = &ext2; + ext2.type = &static_ext; + ext2.dest = &ext2_dest; + ext2.next = NULL; + + stream = pb_istream_from_buffer(buffer, msgsize); + if (!pb_decode(&stream, TestMessage_fields, &msg)) + { + fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&stream)); + return false; + } + + /* Make sure it encodes back to same data */ + { + uint8_t buffer2[256]; + pb_ostream_t ostream = pb_ostream_from_buffer(buffer2, sizeof(buffer2)); + TEST(pb_encode(&ostream, TestMessage_fields, &msg)); + TEST(ostream.bytes_written == msgsize); + TEST(memcmp(buffer, buffer2, msgsize) == 0); + } + + /* Make sure that malloc counters work */ + TEST(get_alloc_count() > 0); + + /* Make sure that pb_release releases everything */ + pb_release(TestMessage_fields, &msg); + TEST(get_alloc_count() == 0); + + /* Check that double-free is a no-op */ + pb_release(TestMessage_fields, &msg); + TEST(get_alloc_count() == 0); + } + + return true; +} + +/* Oneofs */ +static bool test_OneofMessage() +{ + uint8_t buffer[256]; + size_t msgsize; + + { + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Encode first with TestMessage */ + { + OneofMessage msg = OneofMessage_init_zero; + msg.which_msgs = OneofMessage_msg1_tag; + + fill_TestMessage(&msg.msgs.msg1); + + if (!pb_encode(&stream, OneofMessage_fields, &msg)) + { + fprintf(stderr, "Encode failed: %s\n", PB_GET_ERROR(&stream)); + return false; + } + } + + /* Encode second with SubMessage, invoking 'merge' behaviour */ + { + OneofMessage msg = OneofMessage_init_zero; + msg.which_msgs = OneofMessage_msg2_tag; + + msg.first = 999; + msg.msgs.msg2.dynamic_str = "ABCD"; + msg.last = 888; + + if (!pb_encode(&stream, OneofMessage_fields, &msg)) + { + fprintf(stderr, "Encode failed: %s\n", PB_GET_ERROR(&stream)); + return false; + } + } + msgsize = stream.bytes_written; + } + + { + OneofMessage msg = OneofMessage_init_zero; + pb_istream_t stream = pb_istream_from_buffer(buffer, msgsize); + if (!pb_decode(&stream, OneofMessage_fields, &msg)) + { + fprintf(stderr, "Decode failed: %s\n", PB_GET_ERROR(&stream)); + return false; + } + + TEST(msg.first == 999); + TEST(msg.which_msgs == OneofMessage_msg2_tag); + TEST(msg.msgs.msg2.dynamic_str); + TEST(strcmp(msg.msgs.msg2.dynamic_str, "ABCD") == 0); + TEST(msg.msgs.msg2.dynamic_str_arr == NULL); + TEST(msg.msgs.msg2.dynamic_submsg == NULL); + TEST(msg.last == 888); + + pb_release(OneofMessage_fields, &msg); + TEST(get_alloc_count() == 0); + pb_release(OneofMessage_fields, &msg); + TEST(get_alloc_count() == 0); + } + + return true; +} + +int main() +{ + if (test_TestMessage() && test_OneofMessage()) + return 0; + else + return 1; +} + diff --git a/tests/mem_release/mem_release.proto b/tests/mem_release/mem_release.proto new file mode 100644 index 0000000..0816dc2 --- /dev/null +++ b/tests/mem_release/mem_release.proto @@ -0,0 +1,35 @@ +syntax = "proto2"; +import "nanopb.proto"; + +message SubMessage +{ + optional string dynamic_str = 1 [(nanopb).type = FT_POINTER]; + repeated string dynamic_str_arr = 2 [(nanopb).type = FT_POINTER]; + repeated SubMessage dynamic_submsg = 3 [(nanopb).type = FT_POINTER]; +} + +message TestMessage +{ + required SubMessage static_req_submsg = 1 [(nanopb).type = FT_STATIC]; + optional SubMessage dynamic_submsg = 2 [(nanopb).type = FT_POINTER]; + optional SubMessage static_opt_submsg = 3 [(nanopb).type = FT_STATIC]; + repeated SubMessage static_rep_submsg = 4 [(nanopb).type = FT_STATIC, (nanopb).max_count=2]; + extensions 100 to 200; +} + +extend TestMessage +{ + optional SubMessage dynamic_ext = 100 [(nanopb).type = FT_POINTER]; + optional SubMessage static_ext = 101 [(nanopb).type = FT_STATIC]; +} + +message OneofMessage +{ + required int32 first = 1; + oneof msgs + { + TestMessage msg1 = 2; + SubMessage msg2 = 3; + } + required int32 last = 4; +} diff --git a/tests/message_sizes/messages1.proto b/tests/message_sizes/messages1.proto index 48af55a..b66fad7 100644 --- a/tests/message_sizes/messages1.proto +++ b/tests/message_sizes/messages1.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + enum MessageStatus { FAIL = 0; OK = 1; diff --git a/tests/message_sizes/messages2.proto b/tests/message_sizes/messages2.proto index 19fc11e..6761408 100644 --- a/tests/message_sizes/messages2.proto +++ b/tests/message_sizes/messages2.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + import 'nanopb.proto'; import 'messages1.proto'; diff --git a/tests/missing_fields/SConscript b/tests/missing_fields/SConscript index 9926efa..86ba083 100644 --- a/tests/missing_fields/SConscript +++ b/tests/missing_fields/SConscript @@ -3,6 +3,6 @@ Import("env") env.NanopbProto("missing_fields") -test = env.Program(["missing_fields.c", "missing_fields.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_decode.o"]) +test = env.Program(["missing_fields.c", "missing_fields.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_decode.o", "$COMMON/pb_common.o"]) env.RunTest(test) diff --git a/tests/missing_fields/missing_fields.proto b/tests/missing_fields/missing_fields.proto index cbb23ba..cc5e550 100644 --- a/tests/missing_fields/missing_fields.proto +++ b/tests/missing_fields/missing_fields.proto @@ -1,5 +1,7 @@ /* Test for one missing field among many */ +syntax = "proto2"; + message AllFields { required int32 field1 = 1; diff --git a/tests/multiple_files/SConscript b/tests/multiple_files/SConscript index 6b4f6b6..b1281e1 100644 --- a/tests/multiple_files/SConscript +++ b/tests/multiple_files/SConscript @@ -4,10 +4,13 @@ Import("env") incpath = env.Clone() incpath.Append(PROTOCPATH = '#multiple_files') +incpath.Append(CPPPATH = '$BUILD/multiple_files') -incpath.NanopbProto("callbacks") -incpath.NanopbProto("callbacks2") -test = incpath.Program(["test_multiple_files.c", "callbacks.pb.c", "callbacks2.pb.c"]) +incpath.NanopbProto(["multifile1", "multifile1.options"]) +incpath.NanopbProto("multifile2") +incpath.NanopbProto("subdir/multifile2") +test = incpath.Program(["test_multiple_files.c", "multifile1.pb.c", + "multifile2.pb.c", "subdir/multifile2.pb.c"]) env.RunTest(test) diff --git a/tests/multiple_files/callbacks2.proto b/tests/multiple_files/callbacks2.proto deleted file mode 100644 index 9a55e15..0000000 --- a/tests/multiple_files/callbacks2.proto +++ /dev/null @@ -1,9 +0,0 @@ -// Test if including generated header file for this file + implicit include of -// callbacks.pb.h still compiles. Used with test_compiles.c. -import "callbacks.proto"; - -message Callback2Message { - required TestMessage tstmsg = 1; - required SubMessage submsg = 2; -} - diff --git a/tests/multiple_files/multifile1.options b/tests/multiple_files/multifile1.options new file mode 100644 index 0000000..c44d266 --- /dev/null +++ b/tests/multiple_files/multifile1.options @@ -0,0 +1 @@ +StaticMessage.repint32 max_count:5 diff --git a/tests/multiple_files/callbacks.proto b/tests/multiple_files/multifile1.proto index ccd1edd..18f2c67 100644 --- a/tests/multiple_files/callbacks.proto +++ b/tests/multiple_files/multifile1.proto @@ -1,3 +1,5 @@ +syntax = "proto2"; + message SubMessage { optional string stringvalue = 1; repeated int32 int32value = 2; @@ -14,3 +16,19 @@ message TestMessage { repeated string repeatedstring = 6; } +message StaticMessage { + repeated fixed32 repint32 = 1; +} + +enum SignedEnum { + SE_MIN = -128; + SE_MAX = 127; +} + +enum UnsignedEnum { + UE_MIN = 0; + UE_MAX = 255; +} + + + diff --git a/tests/multiple_files/multifile2.proto b/tests/multiple_files/multifile2.proto new file mode 100644 index 0000000..4af45fd --- /dev/null +++ b/tests/multiple_files/multifile2.proto @@ -0,0 +1,22 @@ +// Test if including generated header file for this file + implicit include of +// multifile2.pb.h still compiles. Used with test_compiles.c. +syntax = "proto2"; + +import "multifile1.proto"; + +message Callback2Message { + required TestMessage tstmsg = 1; + required SubMessage submsg = 2; +} + +message OneofMessage { + oneof msgs { + StaticMessage tstmsg = 1; + } +} + +message Enums { + required SignedEnum senum = 1; + required UnsignedEnum uenum = 2; +} + diff --git a/tests/multiple_files/subdir/multifile2.proto b/tests/multiple_files/subdir/multifile2.proto new file mode 100644 index 0000000..847a929 --- /dev/null +++ b/tests/multiple_files/subdir/multifile2.proto @@ -0,0 +1,25 @@ +syntax = "proto2"; + +package subdir; + +import "multifile1.proto"; + +message Callback2Message { + required TestMessage tstmsg = 1; + required SubMessage submsg = 2; +} + +message OneofMessage { + oneof msgs { + StaticMessage tstmsg = 1; + } +} + +message Enums { + required SignedEnum senum = 1; + required UnsignedEnum uenum = 2; +} + +message SubdirMessage { + required int32 foo = 1 [default = 15]; +} diff --git a/tests/multiple_files/test_multiple_files.c b/tests/multiple_files/test_multiple_files.c index 05722dc..70a3e59 100644 --- a/tests/multiple_files/test_multiple_files.c +++ b/tests/multiple_files/test_multiple_files.c @@ -4,9 +4,27 @@ #include <stdio.h> #include <pb_encode.h> -#include "callbacks2.pb.h" +#include "unittests.h" +#include "multifile2.pb.h" +#include "subdir/multifile2.pb.h" int main() { - return 0; + int status = 0; + + /* Test that included file options are properly loaded */ + TEST(OneofMessage_size == 27); + + /* Check that enum signedness is detected properly */ + TEST(PB_LTYPE(Enums_fields[0].type) == PB_LTYPE_VARINT); + TEST(PB_LTYPE(Enums_fields[1].type) == PB_LTYPE_UVARINT); + + /* Test that subdir file is correctly included */ + { + subdir_SubdirMessage foo = subdir_SubdirMessage_init_default; + TEST(foo.foo == 15); + /* TEST(subdir_OneofMessage_size == 27); */ /* TODO: Issue #172 */ + } + + return status; } diff --git a/tests/no_errmsg/SConscript b/tests/no_errmsg/SConscript index ed46705..629bfa6 100644 --- a/tests/no_errmsg/SConscript +++ b/tests/no_errmsg/SConscript @@ -18,10 +18,11 @@ strict = opts.Clone() strict.Append(CFLAGS = strict['CORECFLAGS']) strict.Object("pb_decode_noerr.o", "$NANOPB/pb_decode.c") strict.Object("pb_encode_noerr.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_noerr.o", "$NANOPB/pb_common.c") # Now build and run the test normally. -enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_noerr.o"]) -dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_noerr.o"]) +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_noerr.o", "pb_common_noerr.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_noerr.o", "pb_common_noerr.o"]) env.RunTest(enc) env.RunTest([dec, "encode_alltypes.output"]) diff --git a/tests/no_messages/no_messages.proto b/tests/no_messages/no_messages.proto index 279216b..45bb2e6 100644 --- a/tests/no_messages/no_messages.proto +++ b/tests/no_messages/no_messages.proto @@ -1,5 +1,7 @@ /* Test that a file without any messages works. */ +syntax = "proto2"; + enum Test { First = 1; } diff --git a/tests/oneof/SConscript b/tests/oneof/SConscript new file mode 100644 index 0000000..928ce63 --- /dev/null +++ b/tests/oneof/SConscript @@ -0,0 +1,33 @@ +# Test the 'oneof' feature for generating C unions. + +Import('env') + +import re + +match = None +if 'PROTOC_VERSION' in env: + match = re.search('([0-9]+).([0-9]+).([0-9]+)', env['PROTOC_VERSION']) + +if match: + version = list(map(int, match.groups())) + +# Oneof is supported by protoc >= 2.6.0 +if env.GetOption('clean') or (match and (version[0] > 2 or (version[0] == 2 and version[1] >= 6))): + env.NanopbProto('oneof') + + enc = env.Program(['encode_oneof.c', + 'oneof.pb.c', + '$COMMON/pb_encode.o', + '$COMMON/pb_common.o']) + + dec = env.Program(['decode_oneof.c', + 'oneof.pb.c', + '$COMMON/pb_decode.o', + '$COMMON/pb_common.o']) + + env.RunTest("message1.pb", enc, ARGS = ['1']) + env.RunTest("message1.txt", [dec, 'message1.pb'], ARGS = ['1']) + env.RunTest("message2.pb", enc, ARGS = ['2']) + env.RunTest("message2.txt", [dec, 'message2.pb'], ARGS = ['2']) + env.RunTest("message3.pb", enc, ARGS = ['3']) + env.RunTest("message3.txt", [dec, 'message3.pb'], ARGS = ['3']) diff --git a/tests/oneof/decode_oneof.c b/tests/oneof/decode_oneof.c new file mode 100644 index 0000000..37075cd --- /dev/null +++ b/tests/oneof/decode_oneof.c @@ -0,0 +1,131 @@ +/* Decode a message using oneof fields */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pb_decode.h> +#include "oneof.pb.h" +#include "test_helpers.h" +#include "unittests.h" + +/* Test the 'OneOfMessage' */ +int test_oneof_1(pb_istream_t *stream, int option) +{ + OneOfMessage msg; + int status = 0; + + /* To better catch initialization errors */ + memset(&msg, 0xAA, sizeof(msg)); + + if (!pb_decode(stream, OneOfMessage_fields, &msg)) + { + printf("Decoding failed: %s\n", PB_GET_ERROR(stream)); + return 1; + } + + /* Check that the basic fields work normally */ + TEST(msg.prefix == 123); + TEST(msg.suffix == 321); + + /* Check that we got the right oneof according to command line */ + if (option == 1) + { + TEST(msg.which_values == OneOfMessage_first_tag); + TEST(msg.values.first == 999); + } + else if (option == 2) + { + TEST(msg.which_values == OneOfMessage_second_tag); + TEST(strcmp(msg.values.second, "abcd") == 0); + } + else if (option == 3) + { + TEST(msg.which_values == OneOfMessage_third_tag); + TEST(msg.values.third.array[0] == 1); + TEST(msg.values.third.array[1] == 2); + TEST(msg.values.third.array[2] == 3); + TEST(msg.values.third.array[3] == 4); + TEST(msg.values.third.array[4] == 5); + } + + return status; +} + + +/* Test the 'PlainOneOfMessage' */ +int test_oneof_2(pb_istream_t *stream, int option) +{ + PlainOneOfMessage msg = PlainOneOfMessage_init_zero; + int status = 0; + + if (!pb_decode(stream, PlainOneOfMessage_fields, &msg)) + { + printf("Decoding failed: %s\n", PB_GET_ERROR(stream)); + return 1; + } + + /* Check that we got the right oneof according to command line */ + if (option == 1) + { + TEST(msg.which_values == OneOfMessage_first_tag); + TEST(msg.values.first == 999); + } + else if (option == 2) + { + TEST(msg.which_values == OneOfMessage_second_tag); + TEST(strcmp(msg.values.second, "abcd") == 0); + } + else if (option == 3) + { + TEST(msg.which_values == OneOfMessage_third_tag); + TEST(msg.values.third.array[0] == 1); + TEST(msg.values.third.array[1] == 2); + TEST(msg.values.third.array[2] == 3); + TEST(msg.values.third.array[3] == 4); + TEST(msg.values.third.array[4] == 5); + } + + return status; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[OneOfMessage_size]; + size_t count; + int option; + + if (argc != 2) + { + fprintf(stderr, "Usage: decode_oneof [number]\n"); + return 1; + } + option = atoi(argv[1]); + + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + if (!feof(stdin)) + { + printf("Message does not fit in buffer\n"); + return 1; + } + + { + int status = 0; + pb_istream_t stream; + + stream = pb_istream_from_buffer(buffer, count); + status = test_oneof_1(&stream, option); + + if (status != 0) + return status; + + stream = pb_istream_from_buffer(buffer, count); + status = test_oneof_2(&stream, option); + + if (status != 0) + return status; + } + + return 0; +} diff --git a/tests/oneof/encode_oneof.c b/tests/oneof/encode_oneof.c new file mode 100644 index 0000000..913d2d4 --- /dev/null +++ b/tests/oneof/encode_oneof.c @@ -0,0 +1,64 @@ +/* Encode a message using oneof fields */ + +#include <stdio.h> +#include <stdlib.h> +#include <pb_encode.h> +#include "oneof.pb.h" +#include "test_helpers.h" + +int main(int argc, char **argv) +{ + uint8_t buffer[OneOfMessage_size]; + OneOfMessage msg = OneOfMessage_init_zero; + pb_ostream_t stream; + int option; + + if (argc != 2) + { + fprintf(stderr, "Usage: encode_oneof [number]\n"); + return 1; + } + option = atoi(argv[1]); + + /* Prefix and suffix are used to test that the union does not disturb + * other fields in the same message. */ + msg.prefix = 123; + + /* We encode one of the 'values' fields based on command line argument */ + if (option == 1) + { + msg.which_values = OneOfMessage_first_tag; + msg.values.first = 999; + } + else if (option == 2) + { + msg.which_values = OneOfMessage_second_tag; + strcpy(msg.values.second, "abcd"); + } + else if (option == 3) + { + msg.which_values = OneOfMessage_third_tag; + msg.values.third.array_count = 5; + msg.values.third.array[0] = 1; + msg.values.third.array[1] = 2; + msg.values.third.array[2] = 3; + msg.values.third.array[3] = 4; + msg.values.third.array[4] = 5; + } + + msg.suffix = 321; + + stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + if (pb_encode(&stream, OneOfMessage_fields, &msg)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } +} diff --git a/tests/oneof/oneof.proto b/tests/oneof/oneof.proto new file mode 100644 index 0000000..b4fe56f --- /dev/null +++ b/tests/oneof/oneof.proto @@ -0,0 +1,32 @@ +syntax = "proto2"; + +import 'nanopb.proto'; + +message SubMessage +{ + repeated int32 array = 1 [(nanopb).max_count = 8]; +} + +/* Oneof in a message with other fields */ +message OneOfMessage +{ + required int32 prefix = 1; + oneof values + { + int32 first = 5; + string second = 6 [(nanopb).max_size = 8]; + SubMessage third = 7; + } + required int32 suffix = 99; +} + +/* Oneof in a message by itself */ +message PlainOneOfMessage +{ + oneof values + { + int32 first = 5; + string second = 6 [(nanopb).max_size = 8]; + SubMessage third = 7; + } +}
\ No newline at end of file diff --git a/tests/options/SConscript b/tests/options/SConscript index 89a00fa..215e3bd 100644 --- a/tests/options/SConscript +++ b/tests/options/SConscript @@ -4,6 +4,9 @@ Import("env") env.NanopbProto("options") env.Object('options.pb.c') - env.Match(['options.pb.h', 'options.expected']) +env.NanopbProto("proto3_options") +env.Object('proto3_options.pb.c') +env.Match(['proto3_options.pb.h', 'proto3_options.expected']) + diff --git a/tests/options/options.expected b/tests/options/options.expected index e6179a2..9e47e6a 100644 --- a/tests/options/options.expected +++ b/tests/options/options.expected @@ -1,7 +1,20 @@ char filesize\[20\]; char msgsize\[30\]; char fieldsize\[40\]; +char fieldlen\[41\]; pb_callback_t int32_callback; \sEnumValue1 = 1 Message5_EnumValue1 } pb_packed my_packed_struct; +! skipped_field +! SkippedMessage +#define PB_MSG_103 Message3 +#define PB_MSG_104 Message4 +#define PB_MSG_105 Message5 +#define OPTIONS_MESSAGES \\ +\s+PB_MSG\(103,[0-9]*,Message3\) \\ +\s+PB_MSG\(104,-1,Message4\) \\ +\s+PB_MSG\(105,[0-9]*,Message5\) \\ +#define Message5_msgid 105 +! has_proto3field + diff --git a/tests/options/options.proto b/tests/options/options.proto index b5badcf..c6ca5e2 100644 --- a/tests/options/options.proto +++ b/tests/options/options.proto @@ -2,6 +2,8 @@ * options.expected lists the patterns that are searched for in the output. */ +syntax = "proto2"; + import "nanopb.proto"; // File level options @@ -22,12 +24,15 @@ message Message2 // Field level options message Message3 { + option (nanopb_msgopt).msgid = 103; required string fieldsize = 1 [(nanopb).max_size = 40]; + required string fieldlen = 2 [(nanopb).max_length = 40]; } // Forced callback field message Message4 { + option (nanopb_msgopt).msgid = 104; required int32 int32_callback = 1 [(nanopb).type = FT_CALLBACK]; } @@ -47,6 +52,7 @@ message EnumTest // Short enum names inside message message Message5 { + option (nanopb_msgopt).msgid = 105; enum Enum2 { option (nanopb_enumopt).long_names = false; @@ -63,11 +69,30 @@ message my_packed_struct } // Message with ignored field -// Note: doesn't really test if the field is missing in the output, -// but atleast tests that the output compiles. message Message6 { required int32 field1 = 1; - optional int32 field2 = 2 [(nanopb).type = FT_IGNORE]; + optional int32 skipped_field = 2 [(nanopb).type = FT_IGNORE]; +} + +// Message that is skipped +message SkippedMessage +{ + option (nanopb_msgopt).skip_message = true; + required int32 foo = 1; +} + +// Message with oneof field +message OneofMessage +{ + oneof foo { + int32 bar = 1; + } +} + +// Proto3-style optional field in proto2 file +message Proto3Field +{ + optional int32 proto3field = 1 [(nanopb).proto3 = true]; } diff --git a/tests/options/proto3_options.expected b/tests/options/proto3_options.expected new file mode 100644 index 0000000..cc2f29c --- /dev/null +++ b/tests/options/proto3_options.expected @@ -0,0 +1,4 @@ +! bool has_proto3_default +bool has_proto3_off +! bool has_proto3_on + diff --git a/tests/options/proto3_options.proto b/tests/options/proto3_options.proto new file mode 100644 index 0000000..1017f04 --- /dev/null +++ b/tests/options/proto3_options.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; + +import "nanopb.proto"; + +message Message1 +{ + int32 proto3_default = 1; + int32 proto3_off = 2 [(nanopb).proto3 = false]; + int32 proto3_on = 3 [(nanopb).proto3 = true]; +} + diff --git a/tests/package_name/SConscript b/tests/package_name/SConscript index 8f1b902..4afc503 100644 --- a/tests/package_name/SConscript +++ b/tests/package_name/SConscript @@ -3,14 +3,16 @@ Import("env") -# Build a modified alltypes.proto -def modify_proto(target, source, env): - '''Add a "package test.package;" directive to the beginning of the .proto file.''' - data = open(str(source[0]), 'r').read() - open(str(target[0]), 'w').write("package test.package;\n\n" + data) - return 0 +def set_pkgname(src, dst, pkgname): + data = open(str(src)).read() + placeholder = '// package name placeholder' + assert placeholder in data + data = data.replace(placeholder, 'package %s;' % pkgname) + open(str(dst), 'w').write(data) -env.Command("alltypes.proto", "#alltypes/alltypes.proto", modify_proto) +# Build a modified alltypes.proto +env.Command("alltypes.proto", "#alltypes/alltypes.proto", + lambda target, source, env: set_pkgname(source[0], target[0], 'test.package')) env.Command("alltypes.options", "#alltypes/alltypes.options", Copy("$TARGET", "$SOURCE")) env.NanopbProto(["alltypes", "alltypes.options"]) @@ -29,7 +31,7 @@ def modify_c(target, source, env): env.Command("encode_alltypes.c", "#alltypes/encode_alltypes.c", modify_c) # Encode and compare results to original alltypes testcase -enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o"]) +enc = env.Program(["encode_alltypes.c", "alltypes.pb.c", "$COMMON/pb_encode.o", "$COMMON/pb_common.o"]) refdec = "$BUILD/alltypes/decode_alltypes$PROGSUFFIX" env.RunTest(enc) env.Compare(["encode_alltypes.output", "$BUILD/alltypes/encode_alltypes.output"]) diff --git a/tests/regression/issue_118/SConscript b/tests/regression/issue_118/SConscript new file mode 100644 index 0000000..833d9de --- /dev/null +++ b/tests/regression/issue_118/SConscript @@ -0,0 +1,12 @@ +# Regression test for Issue 118: Short enum names in imported proto files are not honoured + +Import("env") +env = env.Clone() +env.Append(PROTOCPATH = "#regression/issue_118") + +env.NanopbProto("enumdef") +env.Object('enumdef.pb.c') + +env.NanopbProto(["enumuse", "enumdef.proto"]) +env.Object('enumuse.pb.c') + diff --git a/tests/regression/issue_118/enumdef.proto b/tests/regression/issue_118/enumdef.proto new file mode 100644 index 0000000..46845bc --- /dev/null +++ b/tests/regression/issue_118/enumdef.proto @@ -0,0 +1,8 @@ +syntax = "proto2"; + +import 'nanopb.proto'; + +enum MyEnum { + option (nanopb_enumopt).long_names = false; + FOOBAR = 1; +} diff --git a/tests/regression/issue_118/enumuse.proto b/tests/regression/issue_118/enumuse.proto new file mode 100644 index 0000000..4afc452 --- /dev/null +++ b/tests/regression/issue_118/enumuse.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; + +import 'enumdef.proto'; + +message MyMessage { + required MyEnum myenum = 1 [default = FOOBAR]; +} diff --git a/tests/regression/issue_125/SConscript b/tests/regression/issue_125/SConscript new file mode 100644 index 0000000..f2155e6 --- /dev/null +++ b/tests/regression/issue_125/SConscript @@ -0,0 +1,9 @@ +# Regression test for Issue 125: Wrong identifier name for extension fields + +Import("env") + +env.NanopbProto(["extensionbug", "extensionbug.options"]) +env.Object('extensionbug.pb.c') + +env.Match(['extensionbug.pb.h', 'extensionbug.expected']) + diff --git a/tests/regression/issue_125/extensionbug.expected b/tests/regression/issue_125/extensionbug.expected new file mode 100644 index 0000000..fc21335 --- /dev/null +++ b/tests/regression/issue_125/extensionbug.expected @@ -0,0 +1,3 @@ +pb_extension_type_t Message2_extras +uint32_t field2 + diff --git a/tests/regression/issue_125/extensionbug.options b/tests/regression/issue_125/extensionbug.options new file mode 100644 index 0000000..30b464a --- /dev/null +++ b/tests/regression/issue_125/extensionbug.options @@ -0,0 +1,4 @@ +* type:FT_IGNORE + +Message2.extras type:FT_STATIC +Message2.field2 type:FT_STATIC diff --git a/tests/regression/issue_125/extensionbug.proto b/tests/regression/issue_125/extensionbug.proto new file mode 100644 index 0000000..fd1e74f --- /dev/null +++ b/tests/regression/issue_125/extensionbug.proto @@ -0,0 +1,18 @@ +syntax = "proto2"; + +message Message1 +{ + optional uint32 fieldA = 1; + extensions 30 to max; +} + +message Message2 +{ + extend Message1 + { + optional Message2 extras = 30; + } + + optional uint32 field1 = 1; + optional uint32 field2 = 2; +} diff --git a/tests/regression/issue_141/SConscript b/tests/regression/issue_141/SConscript new file mode 100644 index 0000000..b6526be --- /dev/null +++ b/tests/regression/issue_141/SConscript @@ -0,0 +1,8 @@ +# Regression test for issue 141: wrong encoded size #define for oneof messages + +Import("env") + +env.NanopbProto("testproto") +env.Object('testproto.pb.c') +env.Match(['testproto.pb.h', 'testproto.expected']) + diff --git a/tests/regression/issue_141/testproto.expected b/tests/regression/issue_141/testproto.expected new file mode 100644 index 0000000..75bc195 --- /dev/null +++ b/tests/regression/issue_141/testproto.expected @@ -0,0 +1,7 @@ +define SubMessage_size \s* 88 +define OneOfMessage_size \s* 113 +define topMessage_size \s* 70 +define MyMessage1_size \s* 46 +define MyMessage2_size \s* 8 +define MyMessage3_size \s* 5 +define MyMessage4_size \s* 18 diff --git a/tests/regression/issue_141/testproto.proto b/tests/regression/issue_141/testproto.proto new file mode 100644 index 0000000..a445c68 --- /dev/null +++ b/tests/regression/issue_141/testproto.proto @@ -0,0 +1,52 @@ +syntax = "proto2"; + +import 'nanopb.proto'; + +message SubMessage +{ + repeated int32 array = 1 [(nanopb).max_count = 8]; +} + +message OneOfMessage +{ + required int32 prefix = 1; + oneof values + { + int32 first = 5; + string second = 6 [(nanopb).max_size = 8]; + SubMessage third = 7; + } + required int32 suffix = 99; +} + +message topMessage { + required int32 start = 1; + oneof msg { + MyMessage1 msg1 = 2; + MyMessage2 msg2 = 3; + } + required int32 end = 4; +} + +message MyMessage1 { + required uint32 n1 = 1; + required uint32 n2 = 2; + required string s = 3 [(nanopb).max_size = 32]; +} + +message MyMessage2 { + required uint32 num = 1; + required bool b = 2; +} + +message MyMessage3 { + required bool bbb = 1; + required string ss = 2 [(nanopb).max_size = 1]; +} + +message MyMessage4 { + required bool bbbb = 1; + required string sss = 2 [(nanopb).max_size = 2]; + required uint32 num = 3; + required uint32 num2 = 4; +} diff --git a/tests/regression/issue_145/SConscript b/tests/regression/issue_145/SConscript new file mode 100644 index 0000000..0b793a7 --- /dev/null +++ b/tests/regression/issue_145/SConscript @@ -0,0 +1,9 @@ +# Regression test for Issue 145: Allow /* */ and // comments in .options files + +Import("env") + +env.NanopbProto(["comments", "comments.options"]) +env.Object('comments.pb.c') + +env.Match(['comments.pb.h', 'comments.expected']) + diff --git a/tests/regression/issue_145/comments.expected b/tests/regression/issue_145/comments.expected new file mode 100644 index 0000000..7f87458 --- /dev/null +++ b/tests/regression/issue_145/comments.expected @@ -0,0 +1,3 @@ +char foo\[5\]; +char bar\[16\]; + diff --git a/tests/regression/issue_145/comments.options b/tests/regression/issue_145/comments.options new file mode 100644 index 0000000..89959ba --- /dev/null +++ b/tests/regression/issue_145/comments.options @@ -0,0 +1,6 @@ +/* Block comment */ +# Line comment +// Line comment +DummyMessage.foo /* Block comment */ max_size:5 +DummyMessage.bar max_size:16 # Line comment ### + diff --git a/tests/regression/issue_145/comments.proto b/tests/regression/issue_145/comments.proto new file mode 100644 index 0000000..621779f --- /dev/null +++ b/tests/regression/issue_145/comments.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; + +message DummyMessage { + required string foo = 1; + required string bar = 2; +} + diff --git a/tests/regression/issue_166/SConscript b/tests/regression/issue_166/SConscript new file mode 100644 index 0000000..c50b919 --- /dev/null +++ b/tests/regression/issue_166/SConscript @@ -0,0 +1,13 @@ +# Verify that the maximum encoded size is calculated properly +# for enums. + +Import('env') + +env.NanopbProto('enums') + +p = env.Program(["enum_encoded_size.c", + "enums.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_166/enum_encoded_size.c b/tests/regression/issue_166/enum_encoded_size.c new file mode 100644 index 0000000..84e1c7d --- /dev/null +++ b/tests/regression/issue_166/enum_encoded_size.c @@ -0,0 +1,43 @@ +#include <stdio.h> +#include <string.h> +#include <pb_encode.h> +#include "unittests.h" +#include "enums.pb.h" + +int main() +{ + int status = 0; + + uint8_t buf[256]; + SignedMsg msg1; + UnsignedMsg msg2; + pb_ostream_t s; + + { + COMMENT("Test negative value of signed enum"); + /* Negative value should take up the maximum size */ + msg1.value = SignedEnum_SE_MIN; + s = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&s, SignedMsg_fields, &msg1)); + TEST(s.bytes_written == SignedMsg_size); + + COMMENT("Test positive value of signed enum"); + /* Positive value should be smaller */ + msg1.value = SignedEnum_SE_MAX; + s = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&s, SignedMsg_fields, &msg1)); + TEST(s.bytes_written < SignedMsg_size); + } + + { + COMMENT("Test positive value of unsigned enum"); + /* This should take up the maximum size */ + msg2.value = UnsignedEnum_UE_MAX; + s = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&s, UnsignedMsg_fields, &msg2)); + TEST(s.bytes_written == UnsignedMsg_size); + } + + return status; +} + diff --git a/tests/regression/issue_166/enums.proto b/tests/regression/issue_166/enums.proto new file mode 100644 index 0000000..3694804 --- /dev/null +++ b/tests/regression/issue_166/enums.proto @@ -0,0 +1,18 @@ +syntax = "proto2"; + +enum SignedEnum { + SE_MIN = -1; + SE_MAX = 255; +} + +enum UnsignedEnum { + UE_MAX = 65536; +} + +message SignedMsg { + required SignedEnum value = 1; +} + +message UnsignedMsg { + required UnsignedEnum value = 1; +} diff --git a/tests/regression/issue_172/SConscript b/tests/regression/issue_172/SConscript new file mode 100644 index 0000000..49c919e --- /dev/null +++ b/tests/regression/issue_172/SConscript @@ -0,0 +1,16 @@ +# Verify that _size define is generated for messages that have +# includes from another directory. + +Import('env') + +incpath = env.Clone() +incpath.Append(PROTOCPATH="#regression/issue_172/submessage") +incpath.Append(CPPPATH="$BUILD/regression/issue_172/submessage") +incpath.NanopbProto('test') +incpath.NanopbProto(['submessage/submessage', 'submessage/submessage.options']) + +p = incpath.Program(["msg_size.c", + "test.pb.c", + "submessage/submessage.pb.c"]) + + diff --git a/tests/regression/issue_172/msg_size.c b/tests/regression/issue_172/msg_size.c new file mode 100644 index 0000000..be45acb --- /dev/null +++ b/tests/regression/issue_172/msg_size.c @@ -0,0 +1,9 @@ +#include "test.pb.h" + +PB_STATIC_ASSERT(testmessage_size >= 1+1+1+1+16, TESTMESSAGE_SIZE_IS_WRONG) + +int main() +{ + return 0; +} + diff --git a/tests/regression/issue_172/submessage/submessage.options b/tests/regression/issue_172/submessage/submessage.options new file mode 100644 index 0000000..12fb198 --- /dev/null +++ b/tests/regression/issue_172/submessage/submessage.options @@ -0,0 +1 @@ +submessage.data max_size: 16 diff --git a/tests/regression/issue_172/submessage/submessage.proto b/tests/regression/issue_172/submessage/submessage.proto new file mode 100644 index 0000000..ce6804a --- /dev/null +++ b/tests/regression/issue_172/submessage/submessage.proto @@ -0,0 +1,4 @@ +syntax = "proto2"; +message submessage { + required bytes data = 1; +} diff --git a/tests/regression/issue_172/test.proto b/tests/regression/issue_172/test.proto new file mode 100644 index 0000000..fbd97be --- /dev/null +++ b/tests/regression/issue_172/test.proto @@ -0,0 +1,6 @@ +syntax = "proto2"; +import "submessage.proto"; + +message testmessage { + optional submessage sub = 1; +} diff --git a/tests/regression/issue_188/SConscript b/tests/regression/issue_188/SConscript new file mode 100644 index 0000000..6bc3271 --- /dev/null +++ b/tests/regression/issue_188/SConscript @@ -0,0 +1,6 @@ +# Regression test for issue with Enums inside OneOf. + +Import('env') + +env.NanopbProto('oneof') + diff --git a/tests/regression/issue_188/oneof.proto b/tests/regression/issue_188/oneof.proto new file mode 100644 index 0000000..e37f5c0 --- /dev/null +++ b/tests/regression/issue_188/oneof.proto @@ -0,0 +1,29 @@ +syntax = "proto2"; + +message MessageOne +{ + required uint32 one = 1; + required uint32 two = 2; + required uint32 three = 3; + required int32 four = 4; +} + +enum EnumTwo +{ + SOME_ENUM_1 = 1; + SOME_ENUM_2 = 5; + SOME_ENUM_3 = 6; + SOME_ENUM_4 = 9; + SOME_ENUM_5 = 10; + SOME_ENUM_6 = 12; + SOME_ENUM_7 = 39; + SOME_ENUM_8 = 401; +} + +message OneofMessage +{ + oneof payload { + MessageOne message = 1; + EnumTwo enum = 2; + } +} diff --git a/tests/regression/issue_195/SConscript b/tests/regression/issue_195/SConscript new file mode 100644 index 0000000..78326d3 --- /dev/null +++ b/tests/regression/issue_195/SConscript @@ -0,0 +1,10 @@ +# Regression test for Issue 195: Message size not calculated if a submessage includes +# bytes. Basically a non-working #define being generated. + +Import("env") + +env.NanopbProto(["test"]) +env.Object('test.pb.c') + +env.Match(['test.pb.h', 'test.expected']) + diff --git a/tests/regression/issue_195/test.expected b/tests/regression/issue_195/test.expected new file mode 100644 index 0000000..83ea7ab --- /dev/null +++ b/tests/regression/issue_195/test.expected @@ -0,0 +1 @@ +/\* TestMessage_size depends diff --git a/tests/regression/issue_195/test.proto b/tests/regression/issue_195/test.proto new file mode 100644 index 0000000..7a77d69 --- /dev/null +++ b/tests/regression/issue_195/test.proto @@ -0,0 +1,8 @@ +message TestMessage { + required uint32 id = 1; + required bytes payload = 2; +} +message EncapsulatedMessage { + required uint32 id = 1; + required TestMessage test = 2; +} diff --git a/tests/regression/issue_203/SConscript b/tests/regression/issue_203/SConscript new file mode 100644 index 0000000..8b4d6cc --- /dev/null +++ b/tests/regression/issue_203/SConscript @@ -0,0 +1,9 @@ +# Regression test for issue with multiple files generated at once + +Import('env') + +env.Command(['file1.pb.c', 'file1.pb.h', 'file2.pb.c', 'file2.pb.h'], ['file1.proto', 'file2.proto'], + env['NANOPB_PROTO_CMD']) + +env.Object('file1.pb.c') +env.Object('file2.pb.c') diff --git a/tests/regression/issue_203/file1.proto b/tests/regression/issue_203/file1.proto new file mode 100644 index 0000000..dae250b --- /dev/null +++ b/tests/regression/issue_203/file1.proto @@ -0,0 +1,10 @@ +syntax = "proto2"; + +message SubMessage1 { + required int32 foo = 1; +} + +message Message1 { + required SubMessage1 bar = 1; +} + diff --git a/tests/regression/issue_203/file2.proto b/tests/regression/issue_203/file2.proto new file mode 100644 index 0000000..513b0f0 --- /dev/null +++ b/tests/regression/issue_203/file2.proto @@ -0,0 +1,10 @@ +syntax = "proto2"; + +message SubMessage2 { + required int32 foo = 1; +} + +message Message2 { + required SubMessage2 bar = 1; +} + diff --git a/tests/regression/issue_205/SConscript b/tests/regression/issue_205/SConscript new file mode 100644 index 0000000..ed8899d --- /dev/null +++ b/tests/regression/issue_205/SConscript @@ -0,0 +1,14 @@ +# Check that pb_release() correctly handles corrupted size fields of +# static arrays. + +Import('env', 'malloc_env') + +env.NanopbProto('size_corruption') + +p = malloc_env.Program(["size_corruption.c", + "size_corruption.pb.c", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_205/size_corruption.c b/tests/regression/issue_205/size_corruption.c new file mode 100644 index 0000000..08cef45 --- /dev/null +++ b/tests/regression/issue_205/size_corruption.c @@ -0,0 +1,12 @@ +#include "size_corruption.pb.h" +#include <pb_decode.h> + +int main() +{ + MainMessage msg = MainMessage_init_zero; + msg.bar_count = (pb_size_t)-1; + pb_release(MainMessage_fields, &msg); + + return 0; +} + diff --git a/tests/regression/issue_205/size_corruption.proto b/tests/regression/issue_205/size_corruption.proto new file mode 100644 index 0000000..6c9c245 --- /dev/null +++ b/tests/regression/issue_205/size_corruption.proto @@ -0,0 +1,11 @@ +syntax = "proto2"; +import 'nanopb.proto'; + +message SubMessage { + repeated int32 foo = 1 [(nanopb).type = FT_POINTER]; +} + +message MainMessage { + repeated SubMessage bar = 1 [(nanopb).max_count = 5]; +} + diff --git a/tests/regression/issue_227/SConscript b/tests/regression/issue_227/SConscript new file mode 100644 index 0000000..1074124 --- /dev/null +++ b/tests/regression/issue_227/SConscript @@ -0,0 +1,14 @@ +# Regression test for Issue 227:Using proto3 type fields can cause unaligned access +# NOTE: This test will only detect problems when run with clang sanitizer (which +# is done regularly by a jenkins run). + +Import('env') + +env.NanopbProto('unaligned_uint64') + +p = env.Program(["unaligned_uint64.c", + "unaligned_uint64.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_227/unaligned_uint64.c b/tests/regression/issue_227/unaligned_uint64.c new file mode 100644 index 0000000..17c1d77 --- /dev/null +++ b/tests/regression/issue_227/unaligned_uint64.c @@ -0,0 +1,14 @@ +#include "unaligned_uint64.pb.h" +#include <pb_encode.h> + +int main() +{ + uint8_t buf[128]; + pb_ostream_t stream = pb_ostream_from_buffer(buf, sizeof(buf)); + MainMessage msg = MainMessage_init_zero; + msg.bar[0] = 'A'; + pb_encode(&stream, MainMessage_fields, &msg); + + return 0; +} + diff --git a/tests/regression/issue_227/unaligned_uint64.proto b/tests/regression/issue_227/unaligned_uint64.proto new file mode 100644 index 0000000..f0269f6 --- /dev/null +++ b/tests/regression/issue_227/unaligned_uint64.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; +import 'nanopb.proto'; + +message MainMessage { + string foo = 1 [(nanopb).max_size = 3]; + string bar = 2 [(nanopb).max_size = 8]; +} + diff --git a/tests/regression/issue_229/SConscript b/tests/regression/issue_229/SConscript new file mode 100644 index 0000000..b0f8376 --- /dev/null +++ b/tests/regression/issue_229/SConscript @@ -0,0 +1,13 @@ +# Regression test for Issue 229: problem encoding message that has +# multiple oneof fields +Import('env') + +env.NanopbProto('multiple_oneof') + +p = env.Program(["multiple_oneof.c", + "multiple_oneof.pb.c", + "$COMMON/pb_decode.o", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_229/multiple_oneof.c b/tests/regression/issue_229/multiple_oneof.c new file mode 100644 index 0000000..902248d --- /dev/null +++ b/tests/regression/issue_229/multiple_oneof.c @@ -0,0 +1,35 @@ +#include "multiple_oneof.pb.h" +#include <unittests.h> +#include <pb_encode.h> +#include <pb_decode.h> + +int main() +{ + int status = 0; + uint8_t buf[128]; + size_t msglen; + + { + pb_ostream_t stream = pb_ostream_from_buffer(buf, sizeof(buf)); + MainMessage msg = MainMessage_init_zero; + msg.which_oneof1 = MainMessage_oneof1_uint32_tag; + msg.oneof1.oneof1_uint32 = 1234; + msg.which_oneof2 = MainMessage_oneof2_uint32_tag; + msg.oneof2.oneof2_uint32 = 5678; + TEST(pb_encode(&stream, MainMessage_fields, &msg)); + msglen = stream.bytes_written; + } + + { + pb_istream_t stream = pb_istream_from_buffer(buf, msglen); + MainMessage msg = MainMessage_init_zero; + TEST(pb_decode(&stream, MainMessage_fields, &msg)); + TEST(msg.which_oneof1 == MainMessage_oneof1_uint32_tag); + TEST(msg.oneof1.oneof1_uint32 == 1234); + TEST(msg.which_oneof2 == MainMessage_oneof2_uint32_tag); + TEST(msg.oneof2.oneof2_uint32 == 5678); + } + + return status; +} + diff --git a/tests/regression/issue_229/multiple_oneof.proto b/tests/regression/issue_229/multiple_oneof.proto new file mode 100644 index 0000000..22373e1 --- /dev/null +++ b/tests/regression/issue_229/multiple_oneof.proto @@ -0,0 +1,11 @@ +syntax = "proto2"; + +message MainMessage { + oneof oneof1 { + uint32 oneof1_uint32 = 1; + } + oneof oneof2 { + uint32 oneof2_uint32 = 2; + } +} + diff --git a/tests/regression/issue_242/SConscript b/tests/regression/issue_242/SConscript new file mode 100644 index 0000000..000063e --- /dev/null +++ b/tests/regression/issue_242/SConscript @@ -0,0 +1,13 @@ +# Regression test for Issue 242: pb_encode does not encode tag for +# extension fields that is all zeros +Import('env') + +env.NanopbProto('zero_value') + +p = env.Program(["zero_value.c", + "zero_value.pb.c", + "$COMMON/pb_decode.o", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_242/zero_value.c b/tests/regression/issue_242/zero_value.c new file mode 100644 index 0000000..b3d96b7 --- /dev/null +++ b/tests/regression/issue_242/zero_value.c @@ -0,0 +1,51 @@ +#include <unittests.h> +#include <pb_encode.h> +#include <pb_decode.h> +#include <string.h> +#include "zero_value.pb.h" + +int main() +{ + int status = 0; + + COMMENT("Test extension fields with zero values"); + { + uint8_t buffer[256] = {0}; + pb_ostream_t ostream; + int32_t value = 0; + Extendable source = {0}; + + pb_extension_t source_ext = {0}; + source_ext.type = &opt_int32; + source_ext.dest = &value; + source.extensions = &source_ext; + + ostream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + TEST(pb_encode(&ostream, Extendable_fields, &source)); + + TEST(ostream.bytes_written == 2); + TEST(memcmp(buffer, "\x58\x00", 2) == 0); + } + + /* Note: There never was a bug here, but this check is included + * in the regression test because the logic is closely related. + */ + COMMENT("Test pointer fields with zero values"); + { + uint8_t buffer[256] = {0}; + pb_ostream_t ostream; + int32_t value = 0; + PointerMessage source = {0}; + + source.opt_int32 = &value; + + ostream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + TEST(pb_encode(&ostream, PointerMessage_fields, &source)); + + TEST(ostream.bytes_written == 2); + TEST(memcmp(buffer, "\x58\x00", 2) == 0); + } + + return status; +} + diff --git a/tests/regression/issue_242/zero_value.proto b/tests/regression/issue_242/zero_value.proto new file mode 100644 index 0000000..020a39a --- /dev/null +++ b/tests/regression/issue_242/zero_value.proto @@ -0,0 +1,15 @@ +syntax = "proto2"; +import "nanopb.proto"; + +message Extendable { + extensions 10 to 100; +} + +extend Extendable { + optional int32 opt_int32 = 11; +} + +message PointerMessage { + optional int32 opt_int32 = 11 [(nanopb).type = FT_POINTER]; +} + diff --git a/tests/regression/issue_247/SConscript b/tests/regression/issue_247/SConscript new file mode 100644 index 0000000..b41e9f2 --- /dev/null +++ b/tests/regression/issue_247/SConscript @@ -0,0 +1,14 @@ +# Test that pb_check_proto3_default_value() correctly skips padding +# bytes in submessage structures. + +Import("env") + +env.NanopbProto("padding") + +p = env.Program(["padding.c", + "padding.pb.c", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) + +env.RunTest(p) + diff --git a/tests/regression/issue_247/padding.c b/tests/regression/issue_247/padding.c new file mode 100644 index 0000000..8860179 --- /dev/null +++ b/tests/regression/issue_247/padding.c @@ -0,0 +1,32 @@ +#include <pb_encode.h> +#include <unittests.h> +#include <string.h> +#include "padding.pb.h" + +int main() +{ + int status = 0; + + TestMessage msg; + + /* Set padding bytes to garbage */ + memset(&msg, 0xAA, sizeof(msg)); + + /* Set all meaningful fields to 0 */ + msg.submsg.boolfield = false; + msg.submsg.intfield = 0; + + /* Test encoding */ + { + pb_byte_t buf[128] = {0}; + pb_ostream_t stream = pb_ostream_from_buffer(buf, sizeof(buf)); + TEST(pb_encode(&stream, TestMessage_fields, &msg)); + + /* Because all fields have zero values, proto3 encoder + * shouldn't write out anything. */ + TEST(stream.bytes_written == 0); + } + + return status; +} + diff --git a/tests/regression/issue_247/padding.proto b/tests/regression/issue_247/padding.proto new file mode 100644 index 0000000..20bddac --- /dev/null +++ b/tests/regression/issue_247/padding.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +import "nanopb.proto"; + +message SubMessage { + bool boolfield = 1; + int64 intfield = 2; +} + +message TestMessage { + SubMessage submsg = 1; +} + diff --git a/tests/regression/issue_249/SConscript b/tests/regression/issue_249/SConscript new file mode 100644 index 0000000..ba66712 --- /dev/null +++ b/tests/regression/issue_249/SConscript @@ -0,0 +1,12 @@ +# Regression test for Issue 249: proto3 mode pb_decode() corrupts callback fields +Import('env') + +env.NanopbProto('test') + +p = env.Program(["test.c", + "test.pb.c", + "$COMMON/pb_decode.o", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) +env.RunTest(p) + diff --git a/tests/regression/issue_249/test.c b/tests/regression/issue_249/test.c new file mode 100644 index 0000000..a37180f --- /dev/null +++ b/tests/regression/issue_249/test.c @@ -0,0 +1,59 @@ +#include "test.pb.h" +#include <unittests.h> +#include <pb_encode.h> +#include <pb_decode.h> + +static bool write_array(pb_ostream_t *stream, const pb_field_t *field, void * const *arg) +{ + int i; + for (i = 0; i < 5; i++) + { + if (!pb_encode_tag_for_field(stream, field)) + return false; + if (!pb_encode_varint(stream, 1000 + i)) + return false; + } + + return true; +} + +static bool read_array(pb_istream_t *stream, const pb_field_t *field, void **arg) +{ + uint32_t i; + int *sum = *arg; + + if (!pb_decode_varint32(stream, &i)) + return false; + + *sum += i; + + return true; +} + +int main() +{ + int status = 0; + pb_byte_t buf[128] = {0}; + pb_size_t msglen; + + { + MainMessage msg = MainMessage_init_zero; + pb_ostream_t stream = pb_ostream_from_buffer(buf, sizeof(buf)); + msg.submsg.foo.funcs.encode = &write_array; + TEST(pb_encode(&stream, MainMessage_fields, &msg)); + msglen = stream.bytes_written; + } + + { + MainMessage msg = MainMessage_init_zero; + pb_istream_t stream = pb_istream_from_buffer(buf, msglen); + int sum = 0; + msg.submsg.foo.funcs.decode = &read_array; + msg.submsg.foo.arg = ∑ + TEST(pb_decode(&stream, MainMessage_fields, &msg)); + TEST(sum == 1000 + 1001 + 1002 + 1003 + 1004); + } + + return status; +} + diff --git a/tests/regression/issue_249/test.proto b/tests/regression/issue_249/test.proto new file mode 100644 index 0000000..eaa2abd --- /dev/null +++ b/tests/regression/issue_249/test.proto @@ -0,0 +1,10 @@ +syntax = "proto3"; + +message SubMessage { + repeated int32 foo = 1; +} + +message MainMessage { + SubMessage submsg = 1; +} + diff --git a/tests/regression/issue_253/SConscript b/tests/regression/issue_253/SConscript new file mode 100644 index 0000000..5a16948 --- /dev/null +++ b/tests/regression/issue_253/SConscript @@ -0,0 +1,15 @@ +# Regression test for Issue 253: Wrong calculated message maximum size + +Import('env') + +env.NanopbProto('short_array') + +p = env.Program(['short_array.c', + 'short_array.pb.c', + "$COMMON/pb_decode.o", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) + +env.RunTest(p) + + diff --git a/tests/regression/issue_253/short_array.c b/tests/regression/issue_253/short_array.c new file mode 100644 index 0000000..5ed6c3f --- /dev/null +++ b/tests/regression/issue_253/short_array.c @@ -0,0 +1,24 @@ +#include <unittests.h> +#include <pb_encode.h> +#include "short_array.pb.h" + +int main() +{ + int status = 0; + + COMMENT("Test message length calculation for short arrays"); + { + uint8_t buffer[TestMessage_size] = {0}; + pb_ostream_t ostream = pb_ostream_from_buffer(buffer, TestMessage_size); + TestMessage msg = TestMessage_init_zero; + + msg.rep_uint32_count = 1; + msg.rep_uint32[0] = (1 << 31); + + TEST(pb_encode(&ostream, TestMessage_fields, &msg)); + TEST(ostream.bytes_written == TestMessage_size); + } + + return status; +} + diff --git a/tests/regression/issue_253/short_array.proto b/tests/regression/issue_253/short_array.proto new file mode 100644 index 0000000..5a5d8a3 --- /dev/null +++ b/tests/regression/issue_253/short_array.proto @@ -0,0 +1,7 @@ +syntax = "proto2"; +import "nanopb.proto"; + +message TestMessage { + repeated uint32 rep_uint32 = 1 [(nanopb).max_count = 1]; +} + diff --git a/tests/regression/issue_256/SConscript b/tests/regression/issue_256/SConscript new file mode 100644 index 0000000..b2c3e86 --- /dev/null +++ b/tests/regression/issue_256/SConscript @@ -0,0 +1,16 @@ +# Regression test for Issue 256: Proto3 mode skips submessages even when +# later array fields have non-zero value + +Import('env') + +env.NanopbProto('submsg_array') + +p = env.Program(['submsg_array.c', + 'submsg_array.pb.c', + "$COMMON/pb_decode.o", + "$COMMON/pb_encode.o", + "$COMMON/pb_common.o"]) + +env.RunTest(p) + + diff --git a/tests/regression/issue_256/submsg_array.c b/tests/regression/issue_256/submsg_array.c new file mode 100644 index 0000000..c63bd30 --- /dev/null +++ b/tests/regression/issue_256/submsg_array.c @@ -0,0 +1,38 @@ +#include <unittests.h> +#include <pb_encode.h> +#include <pb_decode.h> +#include "submsg_array.pb.h" + +int main() +{ + int status = 0; + + COMMENT("Test encoding for submessage with array"); + { + uint8_t buffer[TestMessage_size] = {0}; + pb_ostream_t ostream = pb_ostream_from_buffer(buffer, TestMessage_size); + TestMessage msg = TestMessage_init_zero; + + msg.submsg.rep_uint32_count = 3; + msg.submsg.rep_uint32[0] = 0; + msg.submsg.rep_uint32[1] = 1; + msg.submsg.rep_uint32[2] = 2; + + TEST(pb_encode(&ostream, TestMessage_fields, &msg)); + TEST(ostream.bytes_written > 0); + + { + pb_istream_t istream = pb_istream_from_buffer(buffer, ostream.bytes_written); + TestMessage msg2 = TestMessage_init_zero; + + TEST(pb_decode(&istream, TestMessage_fields, &msg2)); + TEST(msg2.submsg.rep_uint32_count == 3); + TEST(msg2.submsg.rep_uint32[0] == 0); + TEST(msg2.submsg.rep_uint32[1] == 1); + TEST(msg2.submsg.rep_uint32[2] == 2); + } + } + + return status; +} + diff --git a/tests/regression/issue_256/submsg_array.proto b/tests/regression/issue_256/submsg_array.proto new file mode 100644 index 0000000..4964a05 --- /dev/null +++ b/tests/regression/issue_256/submsg_array.proto @@ -0,0 +1,11 @@ +syntax = "proto3"; +import "nanopb.proto"; + +message SubMessage { + repeated uint32 rep_uint32 = 1 [(nanopb).max_count = 3]; +} + +message TestMessage { + SubMessage submsg = 1; +} + diff --git a/tests/regression/issue_259/SConscript b/tests/regression/issue_259/SConscript new file mode 100644 index 0000000..8340d45 --- /dev/null +++ b/tests/regression/issue_259/SConscript @@ -0,0 +1,22 @@ +# Check that callback fields inside malloc()ed messages +# are correctly initialized. + +Import('env', 'malloc_env') + +env.NanopbProto('callback_pointer') + +p = malloc_env.Program(["callback_pointer.c", + "callback_pointer.pb.c", + "$COMMON/pb_decode_with_malloc.o", + "$COMMON/pb_common_with_malloc.o", + "$COMMON/malloc_wrappers.o"]) + +# Run test under valgrind if available +valgrind = env.WhereIs('valgrind') +kwargs = {} +if valgrind: + kwargs['COMMAND'] = valgrind + kwargs['ARGS'] = ["-q", "--error-exitcode=99", p[0].abspath] + +env.RunTest(p, **kwargs) + diff --git a/tests/regression/issue_259/callback_pointer.c b/tests/regression/issue_259/callback_pointer.c new file mode 100644 index 0000000..f150fbe --- /dev/null +++ b/tests/regression/issue_259/callback_pointer.c @@ -0,0 +1,30 @@ +#include "callback_pointer.pb.h" +#include <unittests.h> +#include <pb_decode.h> + +int main() +{ + int status = 0; + const uint8_t msgdata[] = {0x0A, 0x02, 0x08, 0x7F}; + + MainMessage msg = MainMessage_init_zero; + + { + pb_istream_t stream = pb_istream_from_buffer(msgdata, sizeof(msgdata)); + COMMENT("Running first decode"); + TEST(pb_decode(&stream, MainMessage_fields, &msg)); + pb_release(MainMessage_fields, &msg); + } + + { + pb_istream_t stream = pb_istream_from_buffer(msgdata, sizeof(msgdata)); + COMMENT("Running second decode"); + TEST(pb_decode(&stream, MainMessage_fields, &msg)); + pb_release(MainMessage_fields, &msg); + } + + TEST(get_alloc_count() == 0); + + return status; +} + diff --git a/tests/regression/issue_259/callback_pointer.proto b/tests/regression/issue_259/callback_pointer.proto new file mode 100644 index 0000000..a2d04e4 --- /dev/null +++ b/tests/regression/issue_259/callback_pointer.proto @@ -0,0 +1,11 @@ +syntax = "proto2"; +import 'nanopb.proto'; + +message SubMessage { + optional int32 foo = 1 [(nanopb).type = FT_CALLBACK]; +} + +message MainMessage { + optional SubMessage bar = 1 [(nanopb).type = FT_POINTER]; +} + diff --git a/tests/regression/issue_306/SConscript b/tests/regression/issue_306/SConscript new file mode 100644 index 0000000..d3badec --- /dev/null +++ b/tests/regression/issue_306/SConscript @@ -0,0 +1,7 @@ +# Check that generator gives a warning about large extension field number. + +Import('env') + +env.NanopbProto('large_extension') + +env.Match(['large_extension.pb.c', 'large_extension.expected']) diff --git a/tests/regression/issue_306/large_extension.expected b/tests/regression/issue_306/large_extension.expected new file mode 100644 index 0000000..a90bc32 --- /dev/null +++ b/tests/regression/issue_306/large_extension.expected @@ -0,0 +1 @@ +PB_FIELD_32BIT diff --git a/tests/regression/issue_306/large_extension.proto b/tests/regression/issue_306/large_extension.proto new file mode 100644 index 0000000..c3c8f6f --- /dev/null +++ b/tests/regression/issue_306/large_extension.proto @@ -0,0 +1,10 @@ +syntax = "proto2"; + +message Foo { + extensions 1 to max; +} + +extend Foo { + optional int32 foo_ext = 99999; +} + diff --git a/tests/regression/issue_322/SConscript b/tests/regression/issue_322/SConscript new file mode 100644 index 0000000..ec08e74 --- /dev/null +++ b/tests/regression/issue_322/SConscript @@ -0,0 +1,14 @@ +# Check that default values with special characters are +# correctly handled. + +Import('env') + +env.NanopbProto('defaults') + +p = env.Program(["defaults.c", + "defaults.pb.c", + "$COMMON/pb_decode.o", + "$COMMON/pb_common.o"]) + +env.RunTest(p) + diff --git a/tests/regression/issue_322/defaults.c b/tests/regression/issue_322/defaults.c new file mode 100644 index 0000000..b285754 --- /dev/null +++ b/tests/regression/issue_322/defaults.c @@ -0,0 +1,44 @@ +#include "defaults.pb.h" +#include <unittests.h> +#include <pb_decode.h> + +int check_defaults(const DefaultsMsg *msg) +{ + int status = 0; + + TEST(msg->b1[0] == 0xDE && msg->b1[1] == 0xAD && msg->b1[2] == 0x00 && + msg->b1[3] == 0xBE && msg->b1[4] == 0xEF); + TEST(msg->b2.bytes[0] == 0xDE && msg->b2.bytes[1] == 0xAD && + msg->b2.bytes[2] == 0x00 && msg->b2.bytes[3] == 0xBE && + msg->b2.bytes[4] == 0xEF && msg->b2.size == 5); + TEST(msg->b3.bytes[0] == 0xDE && msg->b3.bytes[1] == 0xAD && + msg->b3.bytes[2] == 0x00 && msg->b3.bytes[3] == 0xBE && + msg->b3.bytes[4] == 0xEF && msg->b2.size == 5); + TEST(msg->s1[0] == (char)0xC3 && msg->s1[1] == (char)0xA4 && + msg->s1[2] == (char)0xC3 && msg->s1[3] == (char)0xB6 && + msg->s1[4] == '\0'); + + return status; +} + +int main() +{ + int status = 0; + + { + DefaultsMsg msg = DefaultsMsg_init_default; + COMMENT("Checking defaults from static initializer"); + status += check_defaults(&msg); + } + + { + DefaultsMsg msg = DefaultsMsg_init_zero; + pb_istream_t empty = {0,0,0}; + pb_decode(&empty, DefaultsMsg_fields, &msg); + COMMENT("Checking defaults set at runtime"); + status += check_defaults(&msg); + } + + return status; +} + diff --git a/tests/regression/issue_322/defaults.proto b/tests/regression/issue_322/defaults.proto new file mode 100644 index 0000000..359344d --- /dev/null +++ b/tests/regression/issue_322/defaults.proto @@ -0,0 +1,11 @@ +syntax = "proto2"; +import 'nanopb.proto'; + +message DefaultsMsg { + optional bytes b1 = 1 [default = "\xDE\xAD\x00\xBE\xEF", (nanopb).max_size = 5, (nanopb).fixed_length=true]; + optional bytes b2 = 2 [default = "\xDE\xAD\x00\xBE\xEF", (nanopb).max_size = 5]; + optional bytes b3 = 3 [default = "\xDE\xAD\000\xBE\xEF", (nanopb).max_size = 15]; + optional string s1 = 4 [default = "รครถ", (nanopb).max_size = 8]; +} + + diff --git a/tests/site_scons/site_init.py b/tests/site_scons/site_init.py index 5fb06d6..56e227b 100644 --- a/tests/site_scons/site_init.py +++ b/tests/site_scons/site_init.py @@ -27,16 +27,16 @@ def add_nanopb_builders(env): if env.has_key('ARGS'): args.extend(env['ARGS']) - print 'Command line: ' + str(args) + print('Command line: ' + str(args)) pipe = subprocess.Popen(args, stdin = infile, stdout = open(str(target[0]), 'w'), stderr = sys.stderr) result = pipe.wait() if result == 0: - print '\033[32m[ OK ]\033[0m Ran ' + args[0] + print('\033[32m[ OK ]\033[0m Ran ' + args[0]) else: - print '\033[31m[FAIL]\033[0m Program ' + args[0] + ' returned ' + str(result) + print('\033[31m[FAIL]\033[0m Program ' + args[0] + ' returned ' + str(result)) return result run_test_builder = Builder(action = run_test, @@ -70,10 +70,10 @@ def add_nanopb_builders(env): data1 = open(str(source[0]), 'rb').read() data2 = open(str(source[1]), 'rb').read() if data1 == data2: - print '\033[32m[ OK ]\033[0m Files equal: ' + str(source[0]) + ' and ' + str(source[1]) + print('\033[32m[ OK ]\033[0m Files equal: ' + str(source[0]) + ' and ' + str(source[1])) return 0 else: - print '\033[31m[FAIL]\033[0m Files differ: ' + str(source[0]) + ' and ' + str(source[1]) + print('\033[31m[FAIL]\033[0m Files differ: ' + str(source[0]) + ' and ' + str(source[1])) return 1 compare_builder = Builder(action = compare_files, @@ -85,11 +85,22 @@ def add_nanopb_builders(env): data = open(str(source[0]), 'rU').read() patterns = open(str(source[1])) for pattern in patterns: - if pattern.strip() and not re.search(pattern.strip(), data, re.MULTILINE): - print '\033[31m[FAIL]\033[0m Pattern not found in ' + str(source[0]) + ': ' + pattern - return 1 + if pattern.strip(): + invert = False + if pattern.startswith('! '): + invert = True + pattern = pattern[2:] + + status = re.search(pattern.strip(), data, re.MULTILINE) + + if not status and not invert: + print('\033[31m[FAIL]\033[0m Pattern not found in ' + str(source[0]) + ': ' + pattern) + return 1 + elif status and invert: + print('\033[31m[FAIL]\033[0m Pattern should not exist, but does in ' + str(source[0]) + ': ' + pattern) + return 1 else: - print '\033[32m[ OK ]\033[0m All patterns found in ' + str(source[0]) + print('\033[32m[ OK ]\033[0m All patterns found in ' + str(source[0])) return 0 match_builder = Builder(action = match_files, suffix = '.matched') diff --git a/tests/site_scons/site_tools/nanopb.py b/tests/site_scons/site_tools/nanopb.py index b3e58fa..c72a45d 100644 --- a/tests/site_scons/site_tools/nanopb.py +++ b/tests/site_scons/site_tools/nanopb.py @@ -118,7 +118,7 @@ def generate(env): env.SetDefault(PROTOCPATH = ['.', os.path.join(env['NANOPB'], 'generator', 'proto')]) - env.SetDefault(NANOPB_PROTO_CMD = '$PROTOC $PROTOC_OPTS --nanopb_out=. $SOURCE') + env.SetDefault(NANOPB_PROTO_CMD = '$PROTOC $PROTOCFLAGS --nanopb_out=. $SOURCES') env['BUILDERS']['NanopbProto'] = _nanopb_proto_builder def exists(env): diff --git a/tests/special_characters/funny-proto+name has.characters.proto b/tests/special_characters/funny-proto+name has.characters.proto index e69de29..26b2cb1 100644 --- a/tests/special_characters/funny-proto+name has.characters.proto +++ b/tests/special_characters/funny-proto+name has.characters.proto @@ -0,0 +1 @@ +syntax="proto2"; diff --git a/tests/splint/SConscript b/tests/splint/SConscript index c1432dd..cd4b5b9 100644 --- a/tests/splint/SConscript +++ b/tests/splint/SConscript @@ -11,3 +11,6 @@ if p: env.Command('pb_encode.splint', '$NANOPB/pb_encode.c', 'splint -f splint/splint.rc $SOURCE 2> $TARGET') + env.Command('pb_common.splint', '$NANOPB/pb_common.c', + 'splint -f splint/splint.rc $SOURCE 2> $TARGET') + diff --git a/tests/splint/splint.rc b/tests/splint/splint.rc index 421f567..e47d3c2 100644 --- a/tests/splint/splint.rc +++ b/tests/splint/splint.rc @@ -2,7 +2,6 @@ +partial +matchanyintegral +strictlib --isoreserved # to be fixed in 0.3 -nullassign -predboolint -predboolptr @@ -28,6 +27,7 @@ -kepttrans -branchstate -immediatetrans +-mustfreefresh # These tests give false positives, compiler typically has # better warnings for these. diff --git a/tests/without_64bit/SConscript b/tests/without_64bit/SConscript new file mode 100644 index 0000000..be415d8 --- /dev/null +++ b/tests/without_64bit/SConscript @@ -0,0 +1,49 @@ +# Run the alltypes test case, but compile with PB_WITHOUT_64BIT. + +Import("env") + +env.NanopbProto(["alltypes", "alltypes.options"]) + +# Define the compilation options +opts = env.Clone() +opts.Append(CPPDEFINES = {'PB_WITHOUT_64BIT': 1, 'HAVE_STDINT_H': 0, 'PB_SYSTEM_HEADER': '\\"no_64bit_syshdr.h\\"'}) +opts.Append(CPPPATH = "#without_64bit") + +if 'SYSHDR' in opts: + opts.Append(CPPDEFINES = {'PB_OLD_SYSHDR': opts['SYSHDR']}) + +# Build new version of core +strict = opts.Clone() +strict.Append(CFLAGS = strict['CORECFLAGS']) +strict.Object("pb_decode_no64bit.o", "$NANOPB/pb_decode.c") +strict.Object("pb_encode_no64bit.o", "$NANOPB/pb_encode.c") +strict.Object("pb_common_no64bit.o", "$NANOPB/pb_common.c") + +# Now build and run the test normally. +enc = opts.Program(["encode_alltypes.c", "alltypes.pb.c", "pb_encode_no64bit.o", "pb_common_no64bit.o"]) +dec = opts.Program(["decode_alltypes.c", "alltypes.pb.c", "pb_decode_no64bit.o", "pb_common_no64bit.o"]) + +env.RunTest(enc) +env.RunTest([dec, "encode_alltypes.output"]) + +# Re-encode the data using protoc, and check that the results from nanopb +# match byte-per-byte to the protoc output. +env.Decode("encode_alltypes.output.decoded", + ["encode_alltypes.output", "alltypes.proto"], + MESSAGE='AllTypes') +env.Encode("encode_alltypes.output.recoded", + ["encode_alltypes.output.decoded", "alltypes.proto"], + MESSAGE='AllTypes') +env.Compare(["encode_alltypes.output", "encode_alltypes.output.recoded"]) + + +# Do the same checks with the optional fields present. +env.RunTest("optionals.output", enc, ARGS = ['1']) +env.RunTest("optionals.decout", [dec, "optionals.output"], ARGS = ['1']) +env.Decode("optionals.output.decoded", + ["optionals.output", "alltypes.proto"], + MESSAGE='AllTypes') +env.Encode("optionals.output.recoded", + ["optionals.output.decoded", "alltypes.proto"], + MESSAGE='AllTypes') +env.Compare(["optionals.output", "optionals.output.recoded"]) diff --git a/tests/without_64bit/alltypes.options b/tests/without_64bit/alltypes.options new file mode 100644 index 0000000..0d5ab12 --- /dev/null +++ b/tests/without_64bit/alltypes.options @@ -0,0 +1,3 @@ +* max_size:16 +* max_count:5 +*.*fbytes fixed_length:true max_size:4 diff --git a/tests/without_64bit/alltypes.proto b/tests/without_64bit/alltypes.proto new file mode 100644 index 0000000..240685b --- /dev/null +++ b/tests/without_64bit/alltypes.proto @@ -0,0 +1,100 @@ +syntax = "proto2"; +// package name placeholder + +message SubMessage { + required string substuff1 = 1 [default = "1"]; + required int32 substuff2 = 2 [default = 2]; + optional fixed32 substuff3 = 3 [default = 3]; +} + +message EmptyMessage { + +} + +enum HugeEnum { + Negative = -2147483647; /* protoc doesn't accept -2147483648 here */ + Positive = 2147483647; +} + +message Limits { + required int32 int32_min = 1 [default = 2147483647]; + required int32 int32_max = 2 [default = -2147483647]; + required uint32 uint32_min = 3 [default = 4294967295]; + required uint32 uint32_max = 4 [default = 0]; + required HugeEnum enum_min = 9 [default = Positive]; + required HugeEnum enum_max = 10 [default = Negative]; +} + +enum MyEnum { + Zero = 0; + First = 1; + Second = 2; + Truth = 42; +} + +message AllTypes { + required int32 req_int32 = 1; + required uint32 req_uint32 = 3; + required sint32 req_sint32 = 5; + required bool req_bool = 7; + + required fixed32 req_fixed32 = 8; + required sfixed32 req_sfixed32= 9; + required float req_float = 10; + + required string req_string = 14; + required bytes req_bytes = 15; + required SubMessage req_submsg = 16; + required MyEnum req_enum = 17; + required EmptyMessage req_emptymsg = 18; + required bytes req_fbytes = 19; + + repeated int32 rep_int32 = 21 [packed = true]; + repeated uint32 rep_uint32 = 23 [packed = true]; + repeated sint32 rep_sint32 = 25 [packed = true]; + repeated bool rep_bool = 27 [packed = true]; + + repeated fixed32 rep_fixed32 = 28 [packed = true]; + repeated sfixed32 rep_sfixed32= 29 [packed = true]; + repeated float rep_float = 30 [packed = true]; + + repeated string rep_string = 34; + repeated bytes rep_bytes = 35; + repeated SubMessage rep_submsg = 36; + repeated MyEnum rep_enum = 37 [packed = true]; + repeated EmptyMessage rep_emptymsg = 38; + repeated bytes rep_fbytes = 39; + + optional int32 opt_int32 = 41 [default = 4041]; + optional uint32 opt_uint32 = 43 [default = 4043]; + optional sint32 opt_sint32 = 45 [default = 4045]; + optional bool opt_bool = 47 [default = false]; + + optional fixed32 opt_fixed32 = 48 [default = 4048]; + optional sfixed32 opt_sfixed32= 49 [default = 4049]; + optional float opt_float = 50 [default = 4050]; + + optional string opt_string = 54 [default = "4054"]; + optional bytes opt_bytes = 55 [default = "4055"]; + optional SubMessage opt_submsg = 56; + optional MyEnum opt_enum = 57 [default = Second]; + optional EmptyMessage opt_emptymsg = 58; + optional bytes opt_fbytes = 59 [default = "4059"]; + + oneof oneof + { + SubMessage oneof_msg1 = 60; + EmptyMessage oneof_msg2 = 61; + } + + // Check that extreme integer values are handled correctly + required Limits req_limits = 98; + + // Just to make sure that the size of the fields has been calculated + // properly, i.e. otherwise a bug in last field might not be detected. + required int32 end = 99; + + + extensions 200 to 255; +} + diff --git a/tests/without_64bit/decode_alltypes.c b/tests/without_64bit/decode_alltypes.c new file mode 100644 index 0000000..6b5ff8e --- /dev/null +++ b/tests/without_64bit/decode_alltypes.c @@ -0,0 +1,185 @@ +/* Tests the decoding of all types. + * This is the counterpart of test_encode3. + * Run e.g. ./test_encode3 | ./test_decode3 + */ + +#include <stdio.h> +#include <string.h> +#include <stdlib.h> +#include <pb_decode.h> +#include "alltypes.pb.h" +#include "test_helpers.h" + +#define TEST(x) if (!(x)) { \ + printf("Test " #x " failed.\n"); \ + return false; \ + } + +/* This function is called once from main(), it handles + the decoding and checks the fields. */ +bool check_alltypes(pb_istream_t *stream, int mode) +{ + /* Uses _init_default to just make sure that it works. */ + AllTypes alltypes = AllTypes_init_default; + + /* Fill with garbage to better detect initialization errors */ + memset(&alltypes, 0xAA, sizeof(alltypes)); + alltypes.extensions = 0; + + if (!pb_decode(stream, AllTypes_fields, &alltypes)) + return false; + + TEST(alltypes.req_int32 == -1001); + TEST(alltypes.req_uint32 == 1003); + TEST(alltypes.req_sint32 == -1005); + TEST(alltypes.req_bool == true); + + TEST(alltypes.req_fixed32 == 1008); + TEST(alltypes.req_sfixed32 == -1009); + TEST(alltypes.req_float == 1010.0f); + + TEST(strcmp(alltypes.req_string, "1014") == 0); + TEST(alltypes.req_bytes.size == 4); + TEST(memcmp(alltypes.req_bytes.bytes, "1015", 4) == 0); + TEST(strcmp(alltypes.req_submsg.substuff1, "1016") == 0); + TEST(alltypes.req_submsg.substuff2 == 1016); + TEST(alltypes.req_submsg.substuff3 == 3); + TEST(alltypes.req_enum == MyEnum_Truth); + TEST(memcmp(alltypes.req_fbytes, "1019", 4) == 0); + + TEST(alltypes.rep_int32_count == 5 && alltypes.rep_int32[4] == -2001 && alltypes.rep_int32[0] == 0); + TEST(alltypes.rep_uint32_count == 5 && alltypes.rep_uint32[4] == 2003 && alltypes.rep_uint32[0] == 0); + TEST(alltypes.rep_sint32_count == 5 && alltypes.rep_sint32[4] == -2005 && alltypes.rep_sint32[0] == 0); + TEST(alltypes.rep_bool_count == 5 && alltypes.rep_bool[4] == true && alltypes.rep_bool[0] == false); + + TEST(alltypes.rep_fixed32_count == 5 && alltypes.rep_fixed32[4] == 2008 && alltypes.rep_fixed32[0] == 0); + TEST(alltypes.rep_sfixed32_count == 5 && alltypes.rep_sfixed32[4] == -2009 && alltypes.rep_sfixed32[0] == 0); + TEST(alltypes.rep_float_count == 5 && alltypes.rep_float[4] == 2010.0f && alltypes.rep_float[0] == 0.0f); + + TEST(alltypes.rep_string_count == 5 && strcmp(alltypes.rep_string[4], "2014") == 0 && alltypes.rep_string[0][0] == '\0'); + TEST(alltypes.rep_bytes_count == 5 && alltypes.rep_bytes[4].size == 4 && alltypes.rep_bytes[0].size == 0); + TEST(memcmp(alltypes.rep_bytes[4].bytes, "2015", 4) == 0); + + TEST(alltypes.rep_submsg_count == 5); + TEST(strcmp(alltypes.rep_submsg[4].substuff1, "2016") == 0 && alltypes.rep_submsg[0].substuff1[0] == '\0'); + TEST(alltypes.rep_submsg[4].substuff2 == 2016 && alltypes.rep_submsg[0].substuff2 == 0); + TEST(alltypes.rep_submsg[4].substuff3 == 2016 && alltypes.rep_submsg[0].substuff3 == 3); + + TEST(alltypes.rep_enum_count == 5 && alltypes.rep_enum[4] == MyEnum_Truth && alltypes.rep_enum[0] == MyEnum_Zero); + TEST(alltypes.rep_emptymsg_count == 5); + TEST(alltypes.rep_fbytes_count == 5); + TEST(alltypes.rep_fbytes[0][0] == 0 && alltypes.rep_fbytes[0][3] == 0); + TEST(memcmp(alltypes.rep_fbytes[4], "2019", 4) == 0); + + if (mode == 0) + { + /* Expect default values */ + TEST(alltypes.has_opt_int32 == false); + TEST(alltypes.opt_int32 == 4041); + TEST(alltypes.has_opt_uint32 == false); + TEST(alltypes.opt_uint32 == 4043); + TEST(alltypes.has_opt_sint32 == false); + TEST(alltypes.opt_sint32 == 4045); + TEST(alltypes.has_opt_bool == false); + TEST(alltypes.opt_bool == false); + + TEST(alltypes.has_opt_fixed32 == false); + TEST(alltypes.opt_fixed32 == 4048); + TEST(alltypes.has_opt_sfixed32 == false); + TEST(alltypes.opt_sfixed32 == 4049); + TEST(alltypes.has_opt_float == false); + TEST(alltypes.opt_float == 4050.0f); + + TEST(alltypes.has_opt_string == false); + TEST(strcmp(alltypes.opt_string, "4054") == 0); + TEST(alltypes.has_opt_bytes == false); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "4055", 4) == 0); + TEST(alltypes.has_opt_submsg == false); + TEST(strcmp(alltypes.opt_submsg.substuff1, "1") == 0); + TEST(alltypes.opt_submsg.substuff2 == 2); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == false); + TEST(alltypes.opt_enum == MyEnum_Second); + TEST(alltypes.has_opt_emptymsg == false); + TEST(alltypes.has_opt_fbytes == false); + TEST(memcmp(alltypes.opt_fbytes, "4059", 4) == 0); + + TEST(alltypes.which_oneof == 0); + } + else + { + /* Expect filled-in values */ + TEST(alltypes.has_opt_int32 == true); + TEST(alltypes.opt_int32 == 3041); + TEST(alltypes.has_opt_uint32 == true); + TEST(alltypes.opt_uint32 == 3043); + TEST(alltypes.has_opt_sint32 == true); + TEST(alltypes.opt_sint32 == 3045); + TEST(alltypes.has_opt_bool == true); + TEST(alltypes.opt_bool == true); + + TEST(alltypes.has_opt_fixed32 == true); + TEST(alltypes.opt_fixed32 == 3048); + TEST(alltypes.has_opt_sfixed32 == true); + TEST(alltypes.opt_sfixed32 == 3049); + TEST(alltypes.has_opt_float == true); + TEST(alltypes.opt_float == 3050.0f); + + TEST(alltypes.has_opt_string == true); + TEST(strcmp(alltypes.opt_string, "3054") == 0); + TEST(alltypes.has_opt_bytes == true); + TEST(alltypes.opt_bytes.size == 4); + TEST(memcmp(alltypes.opt_bytes.bytes, "3055", 4) == 0); + TEST(alltypes.has_opt_submsg == true); + TEST(strcmp(alltypes.opt_submsg.substuff1, "3056") == 0); + TEST(alltypes.opt_submsg.substuff2 == 3056); + TEST(alltypes.opt_submsg.substuff3 == 3); + TEST(alltypes.has_opt_enum == true); + TEST(alltypes.opt_enum == MyEnum_Truth); + TEST(alltypes.has_opt_emptymsg == true); + TEST(alltypes.has_opt_fbytes == true); + TEST(memcmp(alltypes.opt_fbytes, "3059", 4) == 0); + + TEST(alltypes.which_oneof == AllTypes_oneof_msg1_tag); + TEST(strcmp(alltypes.oneof.oneof_msg1.substuff1, "4059") == 0); + TEST(alltypes.oneof.oneof_msg1.substuff2 == 4059); + } + + TEST(alltypes.req_limits.int32_min == INT32_MIN); + TEST(alltypes.req_limits.int32_max == INT32_MAX); + TEST(alltypes.req_limits.uint32_min == 0); + TEST(alltypes.req_limits.uint32_max == UINT32_MAX); + TEST(alltypes.req_limits.enum_min == HugeEnum_Negative); + TEST(alltypes.req_limits.enum_max == HugeEnum_Positive); + + TEST(alltypes.end == 1099); + + return true; +} + +int main(int argc, char **argv) +{ + uint8_t buffer[1024]; + size_t count; + pb_istream_t stream; + + /* Whether to expect the optional values or the default values. */ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Read the data into buffer */ + SET_BINARY_MODE(stdin); + count = fread(buffer, 1, sizeof(buffer), stdin); + + /* Construct a pb_istream_t for reading from the buffer */ + stream = pb_istream_from_buffer(buffer, count); + + /* Decode and print out the stuff */ + if (!check_alltypes(&stream, mode)) + { + printf("Parsing failed: %s\n", PB_GET_ERROR(&stream)); + return 1; + } else { + return 0; + } +} diff --git a/tests/without_64bit/encode_alltypes.c b/tests/without_64bit/encode_alltypes.c new file mode 100644 index 0000000..9fe26f6 --- /dev/null +++ b/tests/without_64bit/encode_alltypes.c @@ -0,0 +1,124 @@ +/* Attempts to test all the datatypes supported by ProtoBuf. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <pb_encode.h> +#include "alltypes.pb.h" +#include "test_helpers.h" + +int main(int argc, char **argv) +{ + int mode = (argc > 1) ? atoi(argv[1]) : 0; + + /* Initialize the structure with constants */ + AllTypes alltypes = AllTypes_init_zero; + + alltypes.req_int32 = -1001; + alltypes.req_uint32 = 1003; + alltypes.req_sint32 = -1005; + alltypes.req_bool = true; + + alltypes.req_fixed32 = 1008; + alltypes.req_sfixed32 = -1009; + alltypes.req_float = 1010.0f; + + strcpy(alltypes.req_string, "1014"); + alltypes.req_bytes.size = 4; + memcpy(alltypes.req_bytes.bytes, "1015", 4); + strcpy(alltypes.req_submsg.substuff1, "1016"); + alltypes.req_submsg.substuff2 = 1016; + alltypes.req_enum = MyEnum_Truth; + memcpy(alltypes.req_fbytes, "1019", 4); + + alltypes.rep_int32_count = 5; alltypes.rep_int32[4] = -2001; + alltypes.rep_uint32_count = 5; alltypes.rep_uint32[4] = 2003; + alltypes.rep_sint32_count = 5; alltypes.rep_sint32[4] = -2005; + alltypes.rep_bool_count = 5; alltypes.rep_bool[4] = true; + + alltypes.rep_fixed32_count = 5; alltypes.rep_fixed32[4] = 2008; + alltypes.rep_sfixed32_count = 5; alltypes.rep_sfixed32[4] = -2009; + alltypes.rep_float_count = 5; alltypes.rep_float[4] = 2010.0f; + + alltypes.rep_string_count = 5; strcpy(alltypes.rep_string[4], "2014"); + alltypes.rep_bytes_count = 5; alltypes.rep_bytes[4].size = 4; + memcpy(alltypes.rep_bytes[4].bytes, "2015", 4); + + alltypes.rep_submsg_count = 5; + strcpy(alltypes.rep_submsg[4].substuff1, "2016"); + alltypes.rep_submsg[4].substuff2 = 2016; + alltypes.rep_submsg[4].has_substuff3 = true; + alltypes.rep_submsg[4].substuff3 = 2016; + + alltypes.rep_enum_count = 5; alltypes.rep_enum[4] = MyEnum_Truth; + alltypes.rep_emptymsg_count = 5; + + alltypes.rep_fbytes_count = 5; + memcpy(alltypes.rep_fbytes[4], "2019", 4); + + alltypes.req_limits.int32_min = INT32_MIN; + alltypes.req_limits.int32_max = INT32_MAX; + alltypes.req_limits.uint32_min = 0; + alltypes.req_limits.uint32_max = UINT32_MAX; + alltypes.req_limits.enum_min = HugeEnum_Negative; + alltypes.req_limits.enum_max = HugeEnum_Positive; + + if (mode != 0) + { + /* Fill in values for optional fields */ + alltypes.has_opt_int32 = true; + alltypes.opt_int32 = 3041; + alltypes.has_opt_uint32 = true; + alltypes.opt_uint32 = 3043; + alltypes.has_opt_sint32 = true; + alltypes.opt_sint32 = 3045; + alltypes.has_opt_bool = true; + alltypes.opt_bool = true; + + alltypes.has_opt_fixed32 = true; + alltypes.opt_fixed32 = 3048; + alltypes.has_opt_sfixed32 = true; + alltypes.opt_sfixed32 = 3049; + alltypes.has_opt_float = true; + alltypes.opt_float = 3050.0f; + + alltypes.has_opt_string = true; + strcpy(alltypes.opt_string, "3054"); + alltypes.has_opt_bytes = true; + alltypes.opt_bytes.size = 4; + memcpy(alltypes.opt_bytes.bytes, "3055", 4); + alltypes.has_opt_submsg = true; + strcpy(alltypes.opt_submsg.substuff1, "3056"); + alltypes.opt_submsg.substuff2 = 3056; + alltypes.has_opt_enum = true; + alltypes.opt_enum = MyEnum_Truth; + alltypes.has_opt_emptymsg = true; + alltypes.has_opt_fbytes = true; + memcpy(alltypes.opt_fbytes, "3059", 4); + + alltypes.which_oneof = AllTypes_oneof_msg1_tag; + strcpy(alltypes.oneof.oneof_msg1.substuff1, "4059"); + alltypes.oneof.oneof_msg1.substuff2 = 4059; + } + + alltypes.end = 1099; + + { + uint8_t buffer[AllTypes_size]; + pb_ostream_t stream = pb_ostream_from_buffer(buffer, sizeof(buffer)); + + /* Now encode it and check if we succeeded. */ + if (pb_encode(&stream, AllTypes_fields, &alltypes)) + { + SET_BINARY_MODE(stdout); + fwrite(buffer, 1, stream.bytes_written, stdout); + return 0; /* Success */ + } + else + { + fprintf(stderr, "Encoding failed: %s\n", PB_GET_ERROR(&stream)); + return 1; /* Failure */ + } + } +} diff --git a/tests/without_64bit/no_64bit_syshdr.h b/tests/without_64bit/no_64bit_syshdr.h new file mode 100644 index 0000000..970178f --- /dev/null +++ b/tests/without_64bit/no_64bit_syshdr.h @@ -0,0 +1,14 @@ +/* This wrapper undefines (u)int64_t */ + +#ifdef PB_OLD_SYSHDR +#include PB_OLD_SYSHDR +#else +#include <stdint.h> +#include <stddef.h> +#include <stdbool.h> +#include <string.h> +#endif + +#define uint64_t disabled_uint64_t +#define int64_t disabled_int64_t + diff --git a/tools/make_linux_package.sh b/tools/make_linux_package.sh index 332c281..0bcba7d 100755 --- a/tools/make_linux_package.sh +++ b/tools/make_linux_package.sh @@ -3,6 +3,8 @@ # Run this script in the top nanopb directory to create a binary package # for Linux users. +# Requires: protobuf, python-protobuf, pyinstaller + set -e set -x @@ -18,20 +20,21 @@ git archive HEAD | tar x -C $DEST # Rebuild the Python .proto files make -BC $DEST/generator/proto -# Make the nanopb generator available as a protoc plugin -cp $DEST/generator/nanopb_generator.py $DEST/generator/protoc-gen-nanopb.py - # Package the Python libraries -( cd $DEST/generator; bbfreeze nanopb_generator.py protoc-gen-nanopb.py ) -mv $DEST/generator/dist $DEST/generator-bin +( cd $DEST/generator; pyinstaller nanopb_generator.py ) +mv $DEST/generator/dist/nanopb_generator $DEST/generator-bin -# Remove temp file -rm $DEST/generator/protoc-gen-nanopb.py +# Remove temp files +rm -rf $DEST/generator/dist $DEST/generator/build $DEST/generator/nanopb_generator.spec + +# Make the nanopb generator available as a protoc plugin +cp $DEST/generator-bin/nanopb_generator $DEST/generator-bin/protoc-gen-nanopb # Package the protoc compiler cp `which protoc` $DEST/generator-bin/protoc.bin LIBPROTOC=$(ldd `which protoc` | grep -o '/.*libprotoc[^ ]*') -cp $LIBPROTOC $DEST/generator-bin/ +LIBPROTOBUF=$(ldd `which protoc` | grep -o '/.*libprotobuf[^ ]*') +cp $LIBPROTOC $LIBPROTOBUF $DEST/generator-bin/ cat > $DEST/generator-bin/protoc << EOF #!/bin/bash SCRIPTDIR=\$(dirname "\$0") @@ -41,6 +44,9 @@ exec "\$SCRIPTDIR/protoc.bin" "\$@" EOF chmod +x $DEST/generator-bin/protoc +# Remove debugging symbols to reduce size of package +( cd $DEST/generator-bin; strip *.so *.so.* ) + # Tar it all up ( cd dist; tar -czf $VERSION.tar.gz $VERSION ) diff --git a/tools/make_mac_package.sh b/tools/make_mac_package.sh index 21cd170..32bba5c 100755 --- a/tools/make_mac_package.sh +++ b/tools/make_mac_package.sh @@ -45,5 +45,5 @@ EOF chmod +x $DEST/generator-bin/protoc # Tar it all up -( cd dist; zip -r $VERSION.zip $VERSION ) +( cd dist; tar -czf $VERSION.tar.gz $VERSION ) diff --git a/tools/set_version.sh b/tools/set_version.sh index e15a859..0adb597 100755 --- a/tools/set_version.sh +++ b/tools/set_version.sh @@ -6,5 +6,9 @@ sed -i -e 's/nanopb_version\s*=\s*"[^"]*"/nanopb_version = "'$1'"/' generator/nanopb_generator.py sed -i -e 's/#define\s*NANOPB_VERSION\s*.*/#define NANOPB_VERSION '$1'/' pb.h +sed -i -e 's/set(\s*nanopb_VERSION_STRING\s*[^)]*)/set(nanopb_VERSION_STRING '$1')/' CMakeLists.txt - +VERSION_ONLY=$(echo $1 | sed 's/nanopb-//') +if [[ $1 != *dev ]] +then sed -i -e 's/"version":\s*"[^"]*"/"version": "'$VERSION_ONLY'"/' library.json +fi |