From fa77f00cb55ecca70b342c5ebd022ef7a7520791 Mon Sep 17 00:00:00 2001 From: Joel Galenson Date: Fri, 2 Apr 2021 11:32:01 -0700 Subject: Upgrade rust/crates/protobuf to 2.22.1 Test: make Change-Id: Ie58392a6f5fd045cb0adab0adbe869383d10c198 --- .cargo_vcs_info.json | 2 +- Android.bp | 2 +- Cargo.toml | 2 +- Cargo.toml.orig | 2 +- METADATA | 8 +- benches/coded_input_stream.rs | 2 +- benches/coded_output_stream.rs | 11 +- out/version.rs | 6 +- src/buf_read_iter.rs | 4 +- src/coded_input_stream.rs | 1004 ++++++++++++++++++ src/coded_output_stream.rs | 705 +++++++++++++ src/descriptor.rs | 4 +- src/lib.rs | 10 +- src/message.rs | 10 +- src/plugin.rs | 4 +- src/rt.rs | 108 +- src/rustproto.rs | 4 +- src/stream.rs | 1803 -------------------------------- src/types.rs | 4 +- src/unknown.rs | 2 +- src/well_known_types/any.rs | 4 +- src/well_known_types/api.rs | 4 +- src/well_known_types/duration.rs | 4 +- src/well_known_types/empty.rs | 4 +- src/well_known_types/field_mask.rs | 4 +- src/well_known_types/source_context.rs | 4 +- src/well_known_types/struct_pb.rs | 4 +- src/well_known_types/timestamp.rs | 4 +- src/well_known_types/type_pb.rs | 4 +- src/well_known_types/wrappers.rs | 4 +- src/wire_format.rs | 105 ++ 31 files changed, 1924 insertions(+), 1918 deletions(-) create mode 100644 src/coded_input_stream.rs create mode 100644 src/coded_output_stream.rs delete mode 100644 src/stream.rs create mode 100644 src/wire_format.rs diff --git a/.cargo_vcs_info.json b/.cargo_vcs_info.json index e1cdbd8..0be67b3 100644 --- a/.cargo_vcs_info.json +++ b/.cargo_vcs_info.json @@ -1,5 +1,5 @@ { "git": { - "sha1": "6dc2d04102a36615e0ca9853390d77712f889b9c" + "sha1": "3eabf141db9d0d69961ba8a1eaf2907c7aeb8ff9" } } diff --git a/Android.bp b/Android.bp index 4aaf259..012442c 100644 --- a/Android.bp +++ b/Android.bp @@ -1,4 +1,5 @@ // This file is generated by cargo2android.py --run --dependencies --device --features=bytes --copy-out --patch=patches/Android.bp.patch. +// Do not modify this file as changes will be overridden on upgrade. package { default_applicable_licenses: ["external_rust_crates_protobuf_license"], @@ -26,7 +27,6 @@ genrule { rust_library { name: "libprotobuf", - // has rustc warnings host_supported: true, crate_name: "protobuf", srcs: [ diff --git a/Cargo.toml b/Cargo.toml index 1af904e..165e125 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,7 +13,7 @@ [package] edition = "2018" name = "protobuf" -version = "2.22.0" +version = "2.22.1" authors = ["Stepan Koltsov "] description = "Rust implementation of Google protocol buffers\n" homepage = "https://github.com/stepancheg/rust-protobuf/" diff --git a/Cargo.toml.orig b/Cargo.toml.orig index 24d73ad..ea70e2c 100644 --- a/Cargo.toml.orig +++ b/Cargo.toml.orig @@ -1,7 +1,7 @@ [package] name = "protobuf" -version = "2.22.0" +version = "2.22.1" authors = ["Stepan Koltsov "] edition = "2018" license = "MIT" diff --git a/METADATA b/METADATA index 62f030f..586a998 100644 --- a/METADATA +++ b/METADATA @@ -7,13 +7,13 @@ third_party { } url { type: ARCHIVE - value: "https://static.crates.io/crates/protobuf/protobuf-2.22.0.crate" + value: "https://static.crates.io/crates/protobuf/protobuf-2.22.1.crate" } - version: "2.22.0" + version: "2.22.1" license_type: NOTICE last_upgrade_date { year: 2021 - month: 2 - day: 9 + month: 4 + day: 2 } } diff --git a/benches/coded_input_stream.rs b/benches/coded_input_stream.rs index ad78115..d1795e5 100644 --- a/benches/coded_input_stream.rs +++ b/benches/coded_input_stream.rs @@ -8,7 +8,7 @@ extern crate test; use std::io; use std::io::Read; -use protobuf::stream::CodedInputStream; +use protobuf::CodedInputStream; use self::test::Bencher; diff --git a/benches/coded_output_stream.rs b/benches/coded_output_stream.rs index 438e980..0f57a38 100644 --- a/benches/coded_output_stream.rs +++ b/benches/coded_output_stream.rs @@ -5,12 +5,11 @@ extern crate protobuf; extern crate test; -use protobuf::stream; - use self::test::Bencher; +use protobuf::CodedOutputStream; #[inline] -fn buffer_write_byte(os: &mut stream::CodedOutputStream) { +fn buffer_write_byte(os: &mut CodedOutputStream) { for i in 0..10 { os.write_raw_byte(test::black_box(i as u8)).unwrap(); } @@ -18,7 +17,7 @@ fn buffer_write_byte(os: &mut stream::CodedOutputStream) { } #[inline] -fn buffer_write_bytes(os: &mut stream::CodedOutputStream) { +fn buffer_write_bytes(os: &mut CodedOutputStream) { for _ in 0..10 { os.write_raw_bytes(test::black_box(b"1234567890")).unwrap(); } @@ -30,7 +29,7 @@ fn bench_buffer(b: &mut Bencher) { b.iter(|| { let mut v = Vec::new(); { - let mut os = stream::CodedOutputStream::new(&mut v); + let mut os = CodedOutputStream::new(&mut v); buffer_write_byte(&mut os); } v @@ -42,7 +41,7 @@ fn bench_buffer_bytes(b: &mut Bencher) { b.iter(|| { let mut v = Vec::new(); { - let mut os = stream::CodedOutputStream::new(&mut v); + let mut os = CodedOutputStream::new(&mut v); buffer_write_bytes(&mut os); } v diff --git a/out/version.rs b/out/version.rs index a242f31..dcf078d 100644 --- a/out/version.rs +++ b/out/version.rs @@ -1,7 +1,7 @@ /// protobuf crate version -pub const VERSION: &'static str = "2.22.0"; +pub const VERSION: &'static str = "2.22.1"; /// This symbol is used by codegen #[doc(hidden)] -pub const VERSION_IDENT: &'static str = "VERSION_2_22_0"; +pub const VERSION_IDENT: &'static str = "VERSION_2_22_1"; /// This symbol can be referenced to assert that proper version of crate is used -pub const VERSION_2_22_0: () = (); +pub const VERSION_2_22_1: () = (); diff --git a/src/buf_read_iter.rs b/src/buf_read_iter.rs index cf36280..21f51a1 100644 --- a/src/buf_read_iter.rs +++ b/src/buf_read_iter.rs @@ -14,13 +14,11 @@ use bytes::Bytes; #[cfg(feature = "bytes")] use bytes::BytesMut; +use crate::coded_input_stream::READ_RAW_BYTES_MAX_ALLOC; use crate::error::WireError; -use crate::stream::READ_RAW_BYTES_MAX_ALLOC; use crate::ProtobufError; use crate::ProtobufResult; -use std::mem::MaybeUninit; - // If an input stream is constructed with a `Read`, we create a // `BufReader` with an internal buffer of this size. const INPUT_STREAM_BUFFER_SIZE: usize = 4096; diff --git a/src/coded_input_stream.rs b/src/coded_input_stream.rs new file mode 100644 index 0000000..52a13a6 --- /dev/null +++ b/src/coded_input_stream.rs @@ -0,0 +1,1004 @@ +#![doc(hidden)] + +//! `CodedInputStream` and `CodedOutputStream` implementations + +use std::io; +use std::io::BufRead; +use std::io::Read; +use std::mem; +use std::slice; + +#[cfg(feature = "bytes")] +use crate::chars::Chars; +#[cfg(feature = "bytes")] +use bytes::Bytes; + +use crate::buf_read_iter::BufReadIter; +use crate::enums::ProtobufEnum; +use crate::error::ProtobufError; +use crate::error::ProtobufResult; +use crate::error::WireError; +use crate::message::Message; +use crate::unknown::UnknownValue; +use crate::wire_format; +use crate::zigzag::decode_zig_zag_32; +use crate::zigzag::decode_zig_zag_64; + +/// Default recursion level limit. 100 is the default value of C++'s implementation. +const DEFAULT_RECURSION_LIMIT: u32 = 100; + +/// Max allocated vec when reading length-delimited from unknown input stream +pub(crate) const READ_RAW_BYTES_MAX_ALLOC: usize = 10_000_000; + +/// Buffered read with handy utilities. +pub struct CodedInputStream<'a> { + source: BufReadIter<'a>, + recursion_level: u32, + recursion_limit: u32, +} + +impl<'a> CodedInputStream<'a> { + /// Wrap a `Read`. + /// + /// Note resulting `CodedInputStream` is buffered even if `Read` is not. + pub fn new(read: &'a mut dyn Read) -> CodedInputStream<'a> { + CodedInputStream::from_buf_read_iter(BufReadIter::from_read(read)) + } + + /// Create from `BufRead`. + /// + /// `CodedInputStream` will utilize `BufRead` buffer. + pub fn from_buffered_reader(buf_read: &'a mut dyn BufRead) -> CodedInputStream<'a> { + CodedInputStream::from_buf_read_iter(BufReadIter::from_buf_read(buf_read)) + } + + /// Read from byte slice + pub fn from_bytes(bytes: &'a [u8]) -> CodedInputStream<'a> { + CodedInputStream::from_buf_read_iter(BufReadIter::from_byte_slice(bytes)) + } + + /// Read from `Bytes`. + /// + /// `CodedInputStream` operations like + /// [`read_carllerche_bytes`](crate::CodedInputStream::read_carllerche_bytes) + /// will return a shared copy of this bytes object. + #[cfg(feature = "bytes")] + pub fn from_carllerche_bytes(bytes: &'a Bytes) -> CodedInputStream<'a> { + CodedInputStream::from_buf_read_iter(BufReadIter::from_bytes(bytes)) + } + + fn from_buf_read_iter(source: BufReadIter<'a>) -> CodedInputStream<'a> { + CodedInputStream { + source: source, + recursion_level: 0, + recursion_limit: DEFAULT_RECURSION_LIMIT, + } + } + + /// Set the recursion limit. + pub fn set_recursion_limit(&mut self, limit: u32) { + self.recursion_limit = limit; + } + + #[inline] + pub(crate) fn incr_recursion(&mut self) -> ProtobufResult<()> { + if self.recursion_level >= self.recursion_limit { + return Err(ProtobufError::WireError(WireError::OverRecursionLimit)); + } + self.recursion_level += 1; + Ok(()) + } + + #[inline] + pub(crate) fn decr_recursion(&mut self) { + self.recursion_level -= 1; + } + + /// How many bytes processed + pub fn pos(&self) -> u64 { + self.source.pos() + } + + /// How many bytes until current limit + pub fn bytes_until_limit(&self) -> u64 { + self.source.bytes_until_limit() + } + + /// Read bytes into given `buf`. + /// + /// Return `0` on EOF. + // TODO: overload with `Read::read` + pub fn read(&mut self, buf: &mut [u8]) -> ProtobufResult<()> { + self.source.read_exact(buf)?; + Ok(()) + } + + /// Read exact number of bytes as `Bytes` object. + /// + /// This operation returns a shared view if `CodedInputStream` is + /// constructed with `Bytes` parameter. + #[cfg(feature = "bytes")] + fn read_raw_callerche_bytes(&mut self, count: usize) -> ProtobufResult { + self.source.read_exact_bytes(count) + } + + /// Read one byte + #[inline(always)] + pub fn read_raw_byte(&mut self) -> ProtobufResult { + self.source.read_byte() + } + + /// Push new limit, return previous limit. + pub fn push_limit(&mut self, limit: u64) -> ProtobufResult { + self.source.push_limit(limit) + } + + /// Restore previous limit. + pub fn pop_limit(&mut self, old_limit: u64) { + self.source.pop_limit(old_limit); + } + + /// Are we at EOF? + #[inline(always)] + pub fn eof(&mut self) -> ProtobufResult { + self.source.eof() + } + + /// Check we are at EOF. + /// + /// Return error if we are not at EOF. + pub fn check_eof(&mut self) -> ProtobufResult<()> { + let eof = self.eof()?; + if !eof { + return Err(ProtobufError::WireError(WireError::UnexpectedEof)); + } + Ok(()) + } + + fn read_raw_varint64_slow(&mut self) -> ProtobufResult { + let mut r: u64 = 0; + let mut i = 0; + loop { + if i == 10 { + return Err(ProtobufError::WireError(WireError::IncorrectVarint)); + } + let b = self.read_raw_byte()?; + // TODO: may overflow if i == 9 + r = r | (((b & 0x7f) as u64) << (i * 7)); + i += 1; + if b < 0x80 { + return Ok(r); + } + } + } + + /// Read varint + #[inline(always)] + pub fn read_raw_varint64(&mut self) -> ProtobufResult { + 'slow: loop { + let ret; + let consume; + + loop { + let rem = self.source.remaining_in_buf(); + + if rem.len() >= 1 { + // most varints are in practice fit in 1 byte + if rem[0] < 0x80 { + ret = rem[0] as u64; + consume = 1; + } else { + // handle case of two bytes too + if rem.len() >= 2 && rem[1] < 0x80 { + ret = (rem[0] & 0x7f) as u64 | (rem[1] as u64) << 7; + consume = 2; + } else if rem.len() >= 10 { + // Read from array when buf at at least 10 bytes, + // max len for varint. + let mut r: u64 = 0; + let mut i: usize = 0; + { + let rem = rem; + loop { + if i == 10 { + return Err(ProtobufError::WireError( + WireError::IncorrectVarint, + )); + } + + let b = if true { + // skip range check + unsafe { *rem.get_unchecked(i) } + } else { + rem[i] + }; + + // TODO: may overflow if i == 9 + r = r | (((b & 0x7f) as u64) << (i * 7)); + i += 1; + if b < 0x80 { + break; + } + } + } + consume = i; + ret = r; + } else { + break 'slow; + } + } + } else { + break 'slow; + } + break; + } + + self.source.consume(consume); + return Ok(ret); + } + + self.read_raw_varint64_slow() + } + + /// Read varint + #[inline(always)] + pub fn read_raw_varint32(&mut self) -> ProtobufResult { + self.read_raw_varint64().map(|v| v as u32) + } + + /// Read little-endian 32-bit integer + pub fn read_raw_little_endian32(&mut self) -> ProtobufResult { + let mut r = 0u32; + let bytes: &mut [u8] = unsafe { + let p: *mut u8 = mem::transmute(&mut r); + slice::from_raw_parts_mut(p, mem::size_of::()) + }; + self.read(bytes)?; + Ok(r.to_le()) + } + + /// Read little-endian 64-bit integer + pub fn read_raw_little_endian64(&mut self) -> ProtobufResult { + let mut r = 0u64; + let bytes: &mut [u8] = unsafe { + let p: *mut u8 = mem::transmute(&mut r); + slice::from_raw_parts_mut(p, mem::size_of::()) + }; + self.read(bytes)?; + Ok(r.to_le()) + } + + /// Read tag + #[inline] + pub fn read_tag(&mut self) -> ProtobufResult { + let v = self.read_raw_varint32()?; + match wire_format::Tag::new(v) { + Some(tag) => Ok(tag), + None => Err(ProtobufError::WireError(WireError::IncorrectTag(v))), + } + } + + /// Read tag, return it is pair (field number, wire type) + #[inline] + pub fn read_tag_unpack(&mut self) -> ProtobufResult<(u32, wire_format::WireType)> { + self.read_tag().map(|t| t.unpack()) + } + + /// Read `double` + pub fn read_double(&mut self) -> ProtobufResult { + let bits = self.read_raw_little_endian64()?; + unsafe { Ok(mem::transmute::(bits)) } + } + + /// Read `float` + pub fn read_float(&mut self) -> ProtobufResult { + let bits = self.read_raw_little_endian32()?; + unsafe { Ok(mem::transmute::(bits)) } + } + + /// Read `int64` + pub fn read_int64(&mut self) -> ProtobufResult { + self.read_raw_varint64().map(|v| v as i64) + } + + /// Read `int32` + pub fn read_int32(&mut self) -> ProtobufResult { + self.read_raw_varint32().map(|v| v as i32) + } + + /// Read `uint64` + pub fn read_uint64(&mut self) -> ProtobufResult { + self.read_raw_varint64() + } + + /// Read `uint32` + pub fn read_uint32(&mut self) -> ProtobufResult { + self.read_raw_varint32() + } + + /// Read `sint64` + pub fn read_sint64(&mut self) -> ProtobufResult { + self.read_uint64().map(decode_zig_zag_64) + } + + /// Read `sint32` + pub fn read_sint32(&mut self) -> ProtobufResult { + self.read_uint32().map(decode_zig_zag_32) + } + + /// Read `fixed64` + pub fn read_fixed64(&mut self) -> ProtobufResult { + self.read_raw_little_endian64() + } + + /// Read `fixed32` + pub fn read_fixed32(&mut self) -> ProtobufResult { + self.read_raw_little_endian32() + } + + /// Read `sfixed64` + pub fn read_sfixed64(&mut self) -> ProtobufResult { + self.read_raw_little_endian64().map(|v| v as i64) + } + + /// Read `sfixed32` + pub fn read_sfixed32(&mut self) -> ProtobufResult { + self.read_raw_little_endian32().map(|v| v as i32) + } + + /// Read `bool` + pub fn read_bool(&mut self) -> ProtobufResult { + self.read_raw_varint32().map(|v| v != 0) + } + + /// Read `enum` as `ProtobufEnum` + pub fn read_enum(&mut self) -> ProtobufResult { + let i = self.read_int32()?; + match ProtobufEnum::from_i32(i) { + Some(e) => Ok(e), + None => Err(ProtobufError::WireError(WireError::InvalidEnumValue(i))), + } + } + + /// Read `repeated` packed `double` + pub fn read_repeated_packed_double_into( + &mut self, + target: &mut Vec, + ) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + + target.reserve((len / 4) as usize); + + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_double()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read `repeated` packed `float` + pub fn read_repeated_packed_float_into(&mut self, target: &mut Vec) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + + target.reserve((len / 4) as usize); + + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_float()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read `repeated` packed `int64` + pub fn read_repeated_packed_int64_into(&mut self, target: &mut Vec) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + let old_limit = self.push_limit(len as u64)?; + while !self.eof()? { + target.push(self.read_int64()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read repeated packed `int32` + pub fn read_repeated_packed_int32_into(&mut self, target: &mut Vec) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_int32()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read repeated packed `uint64` + pub fn read_repeated_packed_uint64_into( + &mut self, + target: &mut Vec, + ) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_uint64()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read repeated packed `uint32` + pub fn read_repeated_packed_uint32_into( + &mut self, + target: &mut Vec, + ) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_uint32()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read repeated packed `sint64` + pub fn read_repeated_packed_sint64_into( + &mut self, + target: &mut Vec, + ) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_sint64()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read repeated packed `sint32` + pub fn read_repeated_packed_sint32_into( + &mut self, + target: &mut Vec, + ) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_sint32()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read repeated packed `fixed64` + pub fn read_repeated_packed_fixed64_into( + &mut self, + target: &mut Vec, + ) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + + target.reserve((len / 8) as usize); + + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_fixed64()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read repeated packed `fixed32` + pub fn read_repeated_packed_fixed32_into( + &mut self, + target: &mut Vec, + ) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + + target.reserve((len / 4) as usize); + + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_fixed32()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read repeated packed `sfixed64` + pub fn read_repeated_packed_sfixed64_into( + &mut self, + target: &mut Vec, + ) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + + target.reserve((len / 8) as usize); + + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_sfixed64()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read repeated packed `sfixed32` + pub fn read_repeated_packed_sfixed32_into( + &mut self, + target: &mut Vec, + ) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + + target.reserve((len / 4) as usize); + + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_sfixed32()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read repeated packed `bool` + pub fn read_repeated_packed_bool_into(&mut self, target: &mut Vec) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + + // regular bool value is 1-byte size + target.reserve(len as usize); + + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_bool()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read repeated packed `enum` into `ProtobufEnum` + pub fn read_repeated_packed_enum_into( + &mut self, + target: &mut Vec, + ) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + let old_limit = self.push_limit(len)?; + while !self.eof()? { + target.push(self.read_enum()?); + } + self.pop_limit(old_limit); + Ok(()) + } + + /// Read `UnknownValue` + pub fn read_unknown( + &mut self, + wire_type: wire_format::WireType, + ) -> ProtobufResult { + match wire_type { + wire_format::WireTypeVarint => { + self.read_raw_varint64().map(|v| UnknownValue::Varint(v)) + } + wire_format::WireTypeFixed64 => self.read_fixed64().map(|v| UnknownValue::Fixed64(v)), + wire_format::WireTypeFixed32 => self.read_fixed32().map(|v| UnknownValue::Fixed32(v)), + wire_format::WireTypeLengthDelimited => { + let len = self.read_raw_varint32()?; + self.read_raw_bytes(len) + .map(|v| UnknownValue::LengthDelimited(v)) + } + _ => Err(ProtobufError::WireError(WireError::UnexpectedWireType( + wire_type, + ))), + } + } + + /// Skip field + pub fn skip_field(&mut self, wire_type: wire_format::WireType) -> ProtobufResult<()> { + self.read_unknown(wire_type).map(|_| ()) + } + + /// Read raw bytes into the supplied vector. The vector will be resized as needed and + /// overwritten. + pub fn read_raw_bytes_into(&mut self, count: u32, target: &mut Vec) -> ProtobufResult<()> { + if false { + // Master uses this version, but keep existing version for a while + // to avoid possible breakages. + return self.source.read_exact_to_vec(count as usize, target); + } + + let count = count as usize; + + // TODO: also do some limits when reading from unlimited source + if count as u64 > self.source.bytes_until_limit() { + return Err(ProtobufError::WireError(WireError::TruncatedMessage)); + } + + unsafe { + target.set_len(0); + } + + if count >= READ_RAW_BYTES_MAX_ALLOC { + // avoid calling `reserve` on buf with very large buffer: could be a malformed message + + let mut take = self.by_ref().take(count as u64); + take.read_to_end(target)?; + + if target.len() != count { + return Err(ProtobufError::WireError(WireError::TruncatedMessage)); + } + } else { + target.reserve(count); + unsafe { + target.set_len(count); + } + + self.source.read_exact(target)?; + } + Ok(()) + } + + /// Read exact number of bytes + pub fn read_raw_bytes(&mut self, count: u32) -> ProtobufResult> { + let mut r = Vec::new(); + self.read_raw_bytes_into(count, &mut r)?; + Ok(r) + } + + /// Skip exact number of bytes + pub fn skip_raw_bytes(&mut self, count: u32) -> ProtobufResult<()> { + // TODO: make it more efficient + self.read_raw_bytes(count).map(|_| ()) + } + + /// Read `bytes` field, length delimited + pub fn read_bytes(&mut self) -> ProtobufResult> { + let mut r = Vec::new(); + self.read_bytes_into(&mut r)?; + Ok(r) + } + + /// Read `bytes` field, length delimited + #[cfg(feature = "bytes")] + pub fn read_carllerche_bytes(&mut self) -> ProtobufResult { + let len = self.read_raw_varint32()?; + self.read_raw_callerche_bytes(len as usize) + } + + /// Read `string` field, length delimited + #[cfg(feature = "bytes")] + pub fn read_carllerche_chars(&mut self) -> ProtobufResult { + let bytes = self.read_carllerche_bytes()?; + Ok(Chars::from_bytes(bytes)?) + } + + /// Read `bytes` field, length delimited + pub fn read_bytes_into(&mut self, target: &mut Vec) -> ProtobufResult<()> { + let len = self.read_raw_varint32()?; + self.read_raw_bytes_into(len, target)?; + Ok(()) + } + + /// Read `string` field, length delimited + pub fn read_string(&mut self) -> ProtobufResult { + let mut r = String::new(); + self.read_string_into(&mut r)?; + Ok(r) + } + + /// Read `string` field, length delimited + pub fn read_string_into(&mut self, target: &mut String) -> ProtobufResult<()> { + target.clear(); + // take target's buffer + let mut vec = mem::replace(target, String::new()).into_bytes(); + self.read_bytes_into(&mut vec)?; + + let s = match String::from_utf8(vec) { + Ok(t) => t, + Err(_) => return Err(ProtobufError::WireError(WireError::Utf8Error)), + }; + *target = s; + Ok(()) + } + + /// Read message, do not check if message is initialized + pub fn merge_message(&mut self, message: &mut M) -> ProtobufResult<()> { + let len = self.read_raw_varint64()?; + let old_limit = self.push_limit(len)?; + message.merge_from(self)?; + self.pop_limit(old_limit); + Ok(()) + } + + /// Read message + pub fn read_message(&mut self) -> ProtobufResult { + let mut r: M = Message::new(); + self.merge_message(&mut r)?; + r.check_initialized()?; + Ok(r) + } +} + +impl<'a> Read for CodedInputStream<'a> { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + self.source.read(buf).map_err(Into::into) + } +} + +impl<'a> BufRead for CodedInputStream<'a> { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.source.fill_buf().map_err(Into::into) + } + + fn consume(&mut self, amt: usize) { + self.source.consume(amt) + } +} + +/// Helper internal utility, should not be used directly +#[doc(hidden)] +pub trait WithCodedInputStream { + fn with_coded_input_stream(self, cb: F) -> ProtobufResult + where + F: FnOnce(&mut CodedInputStream) -> ProtobufResult; +} + +impl<'a> WithCodedInputStream for &'a mut (dyn Read + 'a) { + fn with_coded_input_stream(self, cb: F) -> ProtobufResult + where + F: FnOnce(&mut CodedInputStream) -> ProtobufResult, + { + let mut is = CodedInputStream::new(self); + let r = cb(&mut is)?; + is.check_eof()?; + Ok(r) + } +} + +impl<'a> WithCodedInputStream for &'a mut (dyn BufRead + 'a) { + fn with_coded_input_stream(self, cb: F) -> ProtobufResult + where + F: FnOnce(&mut CodedInputStream) -> ProtobufResult, + { + let mut is = CodedInputStream::from_buffered_reader(self); + let r = cb(&mut is)?; + is.check_eof()?; + Ok(r) + } +} + +impl<'a> WithCodedInputStream for &'a [u8] { + fn with_coded_input_stream(self, cb: F) -> ProtobufResult + where + F: FnOnce(&mut CodedInputStream) -> ProtobufResult, + { + let mut is = CodedInputStream::from_bytes(self); + let r = cb(&mut is)?; + is.check_eof()?; + Ok(r) + } +} + +#[cfg(feature = "bytes")] +impl<'a> WithCodedInputStream for &'a Bytes { + fn with_coded_input_stream(self, cb: F) -> ProtobufResult + where + F: FnOnce(&mut CodedInputStream) -> ProtobufResult, + { + let mut is = CodedInputStream::from_carllerche_bytes(self); + let r = cb(&mut is)?; + is.check_eof()?; + Ok(r) + } +} + +#[cfg(test)] +mod test { + + use std::fmt::Debug; + use std::io; + use std::io::BufRead; + use std::io::Read; + + use crate::error::ProtobufError; + use crate::error::ProtobufResult; + use crate::hex::decode_hex; + + use super::CodedInputStream; + use super::READ_RAW_BYTES_MAX_ALLOC; + + fn test_read_partial(hex: &str, mut callback: F) + where + F: FnMut(&mut CodedInputStream), + { + let d = decode_hex(hex); + let mut reader = io::Cursor::new(d); + let mut is = CodedInputStream::from_buffered_reader(&mut reader as &mut dyn BufRead); + assert_eq!(0, is.pos()); + callback(&mut is); + } + + fn test_read(hex: &str, mut callback: F) + where + F: FnMut(&mut CodedInputStream), + { + let len = decode_hex(hex).len(); + test_read_partial(hex, |reader| { + callback(reader); + assert!(reader.eof().expect("eof")); + assert_eq!(len as u64, reader.pos()); + }); + } + + fn test_read_v(hex: &str, v: V, mut callback: F) + where + F: FnMut(&mut CodedInputStream) -> ProtobufResult, + V: PartialEq + Debug, + { + test_read(hex, |reader| { + assert_eq!(v, callback(reader).unwrap()); + }); + } + + #[test] + fn test_input_stream_read_raw_byte() { + test_read("17", |is| { + assert_eq!(23, is.read_raw_byte().unwrap()); + }); + } + + #[test] + fn test_input_stream_read_raw_varint() { + test_read_v("07", 7, |reader| reader.read_raw_varint32()); + test_read_v("07", 7, |reader| reader.read_raw_varint64()); + + test_read_v("96 01", 150, |reader| reader.read_raw_varint32()); + test_read_v("96 01", 150, |reader| reader.read_raw_varint64()); + + test_read_v( + "ff ff ff ff ff ff ff ff ff 01", + 0xffffffffffffffff, + |reader| reader.read_raw_varint64(), + ); + + test_read_v("ff ff ff ff 0f", 0xffffffff, |reader| { + reader.read_raw_varint32() + }); + test_read_v("ff ff ff ff 0f", 0xffffffff, |reader| { + reader.read_raw_varint64() + }); + } + + #[test] + fn test_input_stream_read_raw_vaint_malformed() { + // varint cannot have length > 10 + test_read_partial("ff ff ff ff ff ff ff ff ff ff 01", |reader| { + let result = reader.read_raw_varint64(); + match result { + // TODO: make an enum variant + Err(ProtobufError::WireError(..)) => (), + _ => panic!(), + } + }); + test_read_partial("ff ff ff ff ff ff ff ff ff ff 01", |reader| { + let result = reader.read_raw_varint32(); + match result { + // TODO: make an enum variant + Err(ProtobufError::WireError(..)) => (), + _ => panic!(), + } + }); + } + + #[test] + fn test_input_stream_read_raw_varint_unexpected_eof() { + test_read_partial("96 97", |reader| { + let result = reader.read_raw_varint32(); + match result { + Err(ProtobufError::WireError(..)) => (), + _ => panic!(), + } + }); + } + + #[test] + fn test_input_stream_read_raw_varint_pos() { + test_read_partial("95 01 98", |reader| { + assert_eq!(149, reader.read_raw_varint32().unwrap()); + assert_eq!(2, reader.pos()); + }); + } + + #[test] + fn test_input_stream_read_int32() { + test_read_v("02", 2, |reader| reader.read_int32()); + } + + #[test] + fn test_input_stream_read_float() { + test_read_v("95 73 13 61", 17e19, |is| is.read_float()); + } + + #[test] + fn test_input_stream_read_double() { + test_read_v("40 d5 ab 68 b3 07 3d 46", 23e29, |is| is.read_double()); + } + + #[test] + fn test_input_stream_skip_raw_bytes() { + test_read("", |reader| { + reader.skip_raw_bytes(0).unwrap(); + }); + test_read("aa bb", |reader| { + reader.skip_raw_bytes(2).unwrap(); + }); + test_read("aa bb cc dd ee ff", |reader| { + reader.skip_raw_bytes(6).unwrap(); + }); + } + + #[test] + fn test_input_stream_read_raw_bytes() { + test_read("", |reader| { + assert_eq!( + Vec::from(&b""[..]), + reader.read_raw_bytes(0).expect("read_raw_bytes") + ); + }) + } + + #[test] + fn test_input_stream_limits() { + test_read("aa bb cc", |is| { + let old_limit = is.push_limit(1).unwrap(); + assert_eq!(1, is.bytes_until_limit()); + let r1 = is.read_raw_bytes(1).unwrap(); + assert_eq!(&[0xaa as u8], &r1[..]); + is.pop_limit(old_limit); + let r2 = is.read_raw_bytes(2).unwrap(); + assert_eq!(&[0xbb as u8, 0xcc], &r2[..]); + }); + } + + #[test] + fn test_input_stream_io_read() { + test_read("aa bb cc", |is| { + let mut buf = [0; 3]; + assert_eq!(Read::read(is, &mut buf).expect("io::Read"), 3); + assert_eq!(buf, [0xaa, 0xbb, 0xcc]); + }); + } + + #[test] + fn test_input_stream_io_bufread() { + test_read("aa bb cc", |is| { + assert_eq!( + BufRead::fill_buf(is).expect("io::BufRead::fill_buf"), + &[0xaa, 0xbb, 0xcc] + ); + BufRead::consume(is, 3); + }); + } + + #[test] + fn test_input_stream_read_raw_bytes_into_huge() { + let mut v = Vec::new(); + for i in 0..READ_RAW_BYTES_MAX_ALLOC + 1000 { + v.push((i % 10) as u8); + } + + let mut slice: &[u8] = v.as_slice(); + + let mut is = CodedInputStream::new(&mut slice); + + let mut buf = Vec::new(); + + is.read_raw_bytes_into(READ_RAW_BYTES_MAX_ALLOC as u32 + 10, &mut buf) + .expect("read"); + + assert_eq!(READ_RAW_BYTES_MAX_ALLOC + 10, buf.len()); + + buf.clear(); + + is.read_raw_bytes_into(1000 - 10, &mut buf).expect("read"); + + assert_eq!(1000 - 10, buf.len()); + + assert!(is.eof().expect("eof")); + } +} diff --git a/src/coded_output_stream.rs b/src/coded_output_stream.rs new file mode 100644 index 0000000..fb7e75c --- /dev/null +++ b/src/coded_output_stream.rs @@ -0,0 +1,705 @@ +use crate::misc::remaining_capacity_as_slice_mut; +use crate::misc::remove_lifetime_mut; +use crate::varint; +use crate::wire_format; +use crate::zigzag::encode_zig_zag_32; +use crate::zigzag::encode_zig_zag_64; +use crate::Message; +use crate::ProtobufEnum; +use crate::ProtobufResult; +use crate::UnknownFields; +use crate::UnknownValueRef; +use std::io; +use std::io::Write; +use std::mem; + +/// Equal to the default buffer size of `BufWriter`, so when +/// `CodedOutputStream` wraps `BufWriter`, it often skips double buffering. +const OUTPUT_STREAM_BUFFER_SIZE: usize = 8 * 1024; + +#[doc(hidden)] +pub trait WithCodedOutputStream { + fn with_coded_output_stream(self, cb: F) -> ProtobufResult + where + F: FnOnce(&mut CodedOutputStream) -> ProtobufResult; +} + +impl<'a> WithCodedOutputStream for &'a mut (dyn Write + 'a) { + fn with_coded_output_stream(self, cb: F) -> ProtobufResult + where + F: FnOnce(&mut CodedOutputStream) -> ProtobufResult, + { + let mut os = CodedOutputStream::new(self); + let r = cb(&mut os)?; + os.flush()?; + Ok(r) + } +} + +impl<'a> WithCodedOutputStream for &'a mut Vec { + fn with_coded_output_stream(mut self, cb: F) -> ProtobufResult + where + F: FnOnce(&mut CodedOutputStream) -> ProtobufResult, + { + let mut os = CodedOutputStream::vec(&mut self); + let r = cb(&mut os)?; + os.flush()?; + Ok(r) + } +} + +#[doc(hidden)] +pub fn with_coded_output_stream_to_bytes(cb: F) -> ProtobufResult> +where + F: FnOnce(&mut CodedOutputStream) -> ProtobufResult<()>, +{ + let mut v = Vec::new(); + v.with_coded_output_stream(cb)?; + Ok(v) +} + +enum OutputTarget<'a> { + Write(&'a mut dyn Write, Vec), + Vec(&'a mut Vec), + Bytes, +} + +/// Buffered write with handy utilities +pub struct CodedOutputStream<'a> { + target: OutputTarget<'a>, + // alias to buf from target + buffer: &'a mut [u8], + // within buffer + position: usize, +} + +impl<'a> CodedOutputStream<'a> { + /// Construct from given `Write`. + /// + /// `CodedOutputStream` is buffered even if `Write` is not + pub fn new(writer: &'a mut dyn Write) -> CodedOutputStream<'a> { + let buffer_len = OUTPUT_STREAM_BUFFER_SIZE; + + let mut buffer_storage = Vec::with_capacity(buffer_len); + unsafe { + buffer_storage.set_len(buffer_len); + } + + let buffer = unsafe { remove_lifetime_mut(&mut buffer_storage as &mut [u8]) }; + + CodedOutputStream { + target: OutputTarget::Write(writer, buffer_storage), + buffer: buffer, + position: 0, + } + } + + /// `CodedOutputStream` which writes directly to bytes. + /// + /// Attempt to write more than bytes capacity results in error. + pub fn bytes(bytes: &'a mut [u8]) -> CodedOutputStream<'a> { + CodedOutputStream { + target: OutputTarget::Bytes, + buffer: bytes, + position: 0, + } + } + + /// `CodedOutputStream` which writes directly to `Vec`. + /// + /// Caller should call `flush` at the end to guarantee vec contains + /// all written data. + pub fn vec(vec: &'a mut Vec) -> CodedOutputStream<'a> { + CodedOutputStream { + target: OutputTarget::Vec(vec), + buffer: &mut [], + position: 0, + } + } + + /// Check if EOF is reached. + /// + /// # Panics + /// + /// If underlying write has no EOF + pub fn check_eof(&self) { + match self.target { + OutputTarget::Bytes => { + assert_eq!(self.buffer.len() as u64, self.position as u64); + } + OutputTarget::Write(..) | OutputTarget::Vec(..) => { + panic!("must not be called with Writer or Vec"); + } + } + } + + fn refresh_buffer(&mut self) -> ProtobufResult<()> { + match self.target { + OutputTarget::Write(ref mut write, _) => { + write.write_all(&self.buffer[0..self.position as usize])?; + self.position = 0; + } + OutputTarget::Vec(ref mut vec) => unsafe { + let vec_len = vec.len(); + assert!(vec_len + self.position <= vec.capacity()); + vec.set_len(vec_len + self.position); + vec.reserve(1); + self.buffer = remove_lifetime_mut(remaining_capacity_as_slice_mut(vec)); + self.position = 0; + }, + OutputTarget::Bytes => { + panic!("refresh_buffer must not be called on CodedOutputStream create from slice"); + } + } + Ok(()) + } + + /// Flush the buffer to underlying write + pub fn flush(&mut self) -> ProtobufResult<()> { + match self.target { + OutputTarget::Bytes => Ok(()), + OutputTarget::Write(..) | OutputTarget::Vec(..) => { + // TODO: must not reserve additional in Vec + self.refresh_buffer() + } + } + } + + /// Write a byte + pub fn write_raw_byte(&mut self, byte: u8) -> ProtobufResult<()> { + if self.position as usize == self.buffer.len() { + self.refresh_buffer()?; + } + self.buffer[self.position as usize] = byte; + self.position += 1; + Ok(()) + } + + /// Write bytes + pub fn write_raw_bytes(&mut self, bytes: &[u8]) -> ProtobufResult<()> { + if bytes.len() <= self.buffer.len() - self.position { + let bottom = self.position as usize; + let top = bottom + (bytes.len() as usize); + self.buffer[bottom..top].copy_from_slice(bytes); + self.position += bytes.len(); + return Ok(()); + } + + self.refresh_buffer()?; + + assert!(self.position == 0); + + if self.position + bytes.len() < self.buffer.len() { + &mut self.buffer[self.position..self.position + bytes.len()].copy_from_slice(bytes); + self.position += bytes.len(); + return Ok(()); + } + + match self.target { + OutputTarget::Bytes => { + unreachable!(); + } + OutputTarget::Write(ref mut write, _) => { + write.write_all(bytes)?; + } + OutputTarget::Vec(ref mut vec) => { + vec.extend(bytes); + unsafe { + self.buffer = remove_lifetime_mut(remaining_capacity_as_slice_mut(vec)); + } + } + } + Ok(()) + } + + /// Write a tag + pub fn write_tag( + &mut self, + field_number: u32, + wire_type: wire_format::WireType, + ) -> ProtobufResult<()> { + self.write_raw_varint32(wire_format::Tag::make(field_number, wire_type).value()) + } + + /// Write varint + pub fn write_raw_varint32(&mut self, value: u32) -> ProtobufResult<()> { + if self.buffer.len() - self.position >= 5 { + // fast path + let len = varint::encode_varint32(value, &mut self.buffer[self.position..]); + self.position += len; + Ok(()) + } else { + // slow path + let buf = &mut [0u8; 5]; + let len = varint::encode_varint32(value, buf); + self.write_raw_bytes(&buf[..len]) + } + } + + /// Write varint + pub fn write_raw_varint64(&mut self, value: u64) -> ProtobufResult<()> { + if self.buffer.len() - self.position >= 10 { + // fast path + let len = varint::encode_varint64(value, &mut self.buffer[self.position..]); + self.position += len; + Ok(()) + } else { + // slow path + let buf = &mut [0u8; 10]; + let len = varint::encode_varint64(value, buf); + self.write_raw_bytes(&buf[..len]) + } + } + + /// Write 32-bit integer little endian + pub fn write_raw_little_endian32(&mut self, value: u32) -> ProtobufResult<()> { + let bytes = unsafe { mem::transmute::<_, [u8; 4]>(value.to_le()) }; + self.write_raw_bytes(&bytes) + } + + /// Write 64-bit integer little endian + pub fn write_raw_little_endian64(&mut self, value: u64) -> ProtobufResult<()> { + let bytes = unsafe { mem::transmute::<_, [u8; 8]>(value.to_le()) }; + self.write_raw_bytes(&bytes) + } + + /// Write `float` + pub fn write_float_no_tag(&mut self, value: f32) -> ProtobufResult<()> { + let bits = unsafe { mem::transmute::(value) }; + self.write_raw_little_endian32(bits) + } + + /// Write `double` + pub fn write_double_no_tag(&mut self, value: f64) -> ProtobufResult<()> { + let bits = unsafe { mem::transmute::(value) }; + self.write_raw_little_endian64(bits) + } + + /// Write `float` field + pub fn write_float(&mut self, field_number: u32, value: f32) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeFixed32)?; + self.write_float_no_tag(value)?; + Ok(()) + } + + /// Write `double` field + pub fn write_double(&mut self, field_number: u32, value: f64) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeFixed64)?; + self.write_double_no_tag(value)?; + Ok(()) + } + + /// Write varint + pub fn write_uint64_no_tag(&mut self, value: u64) -> ProtobufResult<()> { + self.write_raw_varint64(value) + } + + /// Write varint + pub fn write_uint32_no_tag(&mut self, value: u32) -> ProtobufResult<()> { + self.write_raw_varint32(value) + } + + /// Write varint + pub fn write_int64_no_tag(&mut self, value: i64) -> ProtobufResult<()> { + self.write_raw_varint64(value as u64) + } + + /// Write varint + pub fn write_int32_no_tag(&mut self, value: i32) -> ProtobufResult<()> { + self.write_raw_varint64(value as u64) + } + + /// Write zigzag varint + pub fn write_sint64_no_tag(&mut self, value: i64) -> ProtobufResult<()> { + self.write_uint64_no_tag(encode_zig_zag_64(value)) + } + + /// Write zigzag varint + pub fn write_sint32_no_tag(&mut self, value: i32) -> ProtobufResult<()> { + self.write_uint32_no_tag(encode_zig_zag_32(value)) + } + + /// Write `fixed64` + pub fn write_fixed64_no_tag(&mut self, value: u64) -> ProtobufResult<()> { + self.write_raw_little_endian64(value) + } + + /// Write `fixed32` + pub fn write_fixed32_no_tag(&mut self, value: u32) -> ProtobufResult<()> { + self.write_raw_little_endian32(value) + } + + /// Write `sfixed64` + pub fn write_sfixed64_no_tag(&mut self, value: i64) -> ProtobufResult<()> { + self.write_raw_little_endian64(value as u64) + } + + /// Write `sfixed32` + pub fn write_sfixed32_no_tag(&mut self, value: i32) -> ProtobufResult<()> { + self.write_raw_little_endian32(value as u32) + } + + /// Write `bool` + pub fn write_bool_no_tag(&mut self, value: bool) -> ProtobufResult<()> { + self.write_raw_varint32(if value { 1 } else { 0 }) + } + + /// Write `enum` + pub fn write_enum_no_tag(&mut self, value: i32) -> ProtobufResult<()> { + self.write_int32_no_tag(value) + } + + /// Write `enum` + pub fn write_enum_obj_no_tag(&mut self, value: E) -> ProtobufResult<()> + where + E: ProtobufEnum, + { + self.write_enum_no_tag(value.value()) + } + + /// Write unknown value + pub fn write_unknown_no_tag(&mut self, unknown: UnknownValueRef) -> ProtobufResult<()> { + match unknown { + UnknownValueRef::Fixed64(fixed64) => self.write_raw_little_endian64(fixed64), + UnknownValueRef::Fixed32(fixed32) => self.write_raw_little_endian32(fixed32), + UnknownValueRef::Varint(varint) => self.write_raw_varint64(varint), + UnknownValueRef::LengthDelimited(bytes) => self.write_bytes_no_tag(bytes), + } + } + + /// Write `uint64` field + pub fn write_uint64(&mut self, field_number: u32, value: u64) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeVarint)?; + self.write_uint64_no_tag(value)?; + Ok(()) + } + + /// Write `uint32` field + pub fn write_uint32(&mut self, field_number: u32, value: u32) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeVarint)?; + self.write_uint32_no_tag(value)?; + Ok(()) + } + + /// Write `int64` field + pub fn write_int64(&mut self, field_number: u32, value: i64) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeVarint)?; + self.write_int64_no_tag(value)?; + Ok(()) + } + + /// Write `int32` field + pub fn write_int32(&mut self, field_number: u32, value: i32) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeVarint)?; + self.write_int32_no_tag(value)?; + Ok(()) + } + + /// Write `sint64` field + pub fn write_sint64(&mut self, field_number: u32, value: i64) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeVarint)?; + self.write_sint64_no_tag(value)?; + Ok(()) + } + + /// Write `sint32` field + pub fn write_sint32(&mut self, field_number: u32, value: i32) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeVarint)?; + self.write_sint32_no_tag(value)?; + Ok(()) + } + + /// Write `fixed64` field + pub fn write_fixed64(&mut self, field_number: u32, value: u64) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeFixed64)?; + self.write_fixed64_no_tag(value)?; + Ok(()) + } + + /// Write `fixed32` field + pub fn write_fixed32(&mut self, field_number: u32, value: u32) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeFixed32)?; + self.write_fixed32_no_tag(value)?; + Ok(()) + } + + /// Write `sfixed64` field + pub fn write_sfixed64(&mut self, field_number: u32, value: i64) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeFixed64)?; + self.write_sfixed64_no_tag(value)?; + Ok(()) + } + + /// Write `sfixed32` field + pub fn write_sfixed32(&mut self, field_number: u32, value: i32) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeFixed32)?; + self.write_sfixed32_no_tag(value)?; + Ok(()) + } + + /// Write `bool` field + pub fn write_bool(&mut self, field_number: u32, value: bool) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeVarint)?; + self.write_bool_no_tag(value)?; + Ok(()) + } + + /// Write `enum` field + pub fn write_enum(&mut self, field_number: u32, value: i32) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeVarint)?; + self.write_enum_no_tag(value)?; + Ok(()) + } + + /// Write `enum` field + pub fn write_enum_obj(&mut self, field_number: u32, value: E) -> ProtobufResult<()> + where + E: ProtobufEnum, + { + self.write_enum(field_number, value.value()) + } + + /// Write unknown field + pub fn write_unknown( + &mut self, + field_number: u32, + value: UnknownValueRef, + ) -> ProtobufResult<()> { + self.write_tag(field_number, value.wire_type())?; + self.write_unknown_no_tag(value)?; + Ok(()) + } + + /// Write unknown fields + pub fn write_unknown_fields(&mut self, fields: &UnknownFields) -> ProtobufResult<()> { + for (number, values) in fields { + for value in values { + self.write_unknown(number, value)?; + } + } + Ok(()) + } + + /// Write bytes + pub fn write_bytes_no_tag(&mut self, bytes: &[u8]) -> ProtobufResult<()> { + self.write_raw_varint32(bytes.len() as u32)?; + self.write_raw_bytes(bytes)?; + Ok(()) + } + + /// Write string + pub fn write_string_no_tag(&mut self, s: &str) -> ProtobufResult<()> { + self.write_bytes_no_tag(s.as_bytes()) + } + + /// Write message + pub fn write_message_no_tag(&mut self, msg: &M) -> ProtobufResult<()> { + msg.write_length_delimited_to(self) + } + + /// Write `bytes` field + pub fn write_bytes(&mut self, field_number: u32, bytes: &[u8]) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeLengthDelimited)?; + self.write_bytes_no_tag(bytes)?; + Ok(()) + } + + /// Write `string` field + pub fn write_string(&mut self, field_number: u32, s: &str) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeLengthDelimited)?; + self.write_string_no_tag(s)?; + Ok(()) + } + + /// Write `message` field + pub fn write_message(&mut self, field_number: u32, msg: &M) -> ProtobufResult<()> { + self.write_tag(field_number, wire_format::WireTypeLengthDelimited)?; + self.write_message_no_tag(msg)?; + Ok(()) + } +} + +impl<'a> Write for CodedOutputStream<'a> { + fn write(&mut self, buf: &[u8]) -> io::Result { + self.write_raw_bytes(buf)?; + Ok(buf.len()) + } + + fn flush(&mut self) -> io::Result<()> { + CodedOutputStream::flush(self).map_err(Into::into) + } +} + +#[cfg(test)] +mod test { + use crate::coded_output_stream::CodedOutputStream; + use crate::hex::decode_hex; + use crate::hex::encode_hex; + use crate::wire_format; + use crate::ProtobufResult; + use std::io::Write; + use std::iter; + + fn test_write(expected: &str, mut gen: F) + where + F: FnMut(&mut CodedOutputStream) -> ProtobufResult<()>, + { + let expected_bytes = decode_hex(expected); + + // write to Write + { + let mut v = Vec::new(); + { + let mut os = CodedOutputStream::new(&mut v as &mut dyn Write); + gen(&mut os).unwrap(); + os.flush().unwrap(); + } + assert_eq!(encode_hex(&expected_bytes), encode_hex(&v)); + } + + // write to &[u8] + { + let mut r = Vec::with_capacity(expected_bytes.len()); + r.resize(expected_bytes.len(), 0); + { + let mut os = CodedOutputStream::bytes(&mut r); + gen(&mut os).unwrap(); + os.check_eof(); + } + assert_eq!(encode_hex(&expected_bytes), encode_hex(&r)); + } + + // write to Vec + { + let mut r = Vec::new(); + r.extend(&[11, 22, 33, 44, 55, 66, 77]); + { + let mut os = CodedOutputStream::vec(&mut r); + gen(&mut os).unwrap(); + os.flush().unwrap(); + } + + r.drain(..7); + assert_eq!(encode_hex(&expected_bytes), encode_hex(&r)); + } + } + + #[test] + fn test_output_stream_write_raw_byte() { + test_write("a1", |os| os.write_raw_byte(0xa1)); + } + + #[test] + fn test_output_stream_write_tag() { + test_write("08", |os| os.write_tag(1, wire_format::WireTypeVarint)); + } + + #[test] + fn test_output_stream_write_raw_bytes() { + test_write("00 ab", |os| os.write_raw_bytes(&[0x00, 0xab])); + + let expected = iter::repeat("01 02 03 04") + .take(2048) + .collect::>() + .join(" "); + test_write(&expected, |os| { + for _ in 0..2048 { + os.write_raw_bytes(&[0x01, 0x02, 0x03, 0x04])?; + } + + Ok(()) + }); + } + + #[test] + fn test_output_stream_write_raw_varint32() { + test_write("96 01", |os| os.write_raw_varint32(150)); + test_write("ff ff ff ff 0f", |os| os.write_raw_varint32(0xffffffff)); + } + + #[test] + fn test_output_stream_write_raw_varint64() { + test_write("96 01", |os| os.write_raw_varint64(150)); + test_write("ff ff ff ff ff ff ff ff ff 01", |os| { + os.write_raw_varint64(0xffffffffffffffff) + }); + } + + #[test] + fn test_output_stream_write_int32_no_tag() { + test_write("ff ff ff ff ff ff ff ff ff 01", |os| { + os.write_int32_no_tag(-1) + }); + } + + #[test] + fn test_output_stream_write_int64_no_tag() { + test_write("ff ff ff ff ff ff ff ff ff 01", |os| { + os.write_int64_no_tag(-1) + }); + } + + #[test] + fn test_output_stream_write_raw_little_endian32() { + test_write("f1 e2 d3 c4", |os| os.write_raw_little_endian32(0xc4d3e2f1)); + } + + #[test] + fn test_output_stream_write_float_no_tag() { + test_write("95 73 13 61", |os| os.write_float_no_tag(17e19)); + } + + #[test] + fn test_output_stream_write_double_no_tag() { + test_write("40 d5 ab 68 b3 07 3d 46", |os| { + os.write_double_no_tag(23e29) + }); + } + + #[test] + fn test_output_stream_write_raw_little_endian64() { + test_write("f1 e2 d3 c4 b5 a6 07 f8", |os| { + os.write_raw_little_endian64(0xf807a6b5c4d3e2f1) + }); + } + + #[test] + fn test_output_stream_io_write() { + let expected = [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77]; + + // write to Write + { + let mut v = Vec::new(); + { + let mut os = CodedOutputStream::new(&mut v as &mut dyn Write); + Write::write(&mut os, &expected).expect("io::Write::write"); + Write::flush(&mut os).expect("io::Write::flush"); + } + assert_eq!(expected, *v); + } + + // write to &[u8] + { + let mut v = Vec::with_capacity(expected.len()); + v.resize(expected.len(), 0); + { + let mut os = CodedOutputStream::bytes(&mut v); + Write::write(&mut os, &expected).expect("io::Write::write"); + Write::flush(&mut os).expect("io::Write::flush"); + os.check_eof(); + } + assert_eq!(expected, *v); + } + + // write to Vec + { + let mut v = Vec::new(); + { + let mut os = CodedOutputStream::vec(&mut v); + Write::write(&mut os, &expected).expect("io::Write::write"); + Write::flush(&mut os).expect("io::Write::flush"); + } + assert_eq!(expected, *v); + } + } +} diff --git a/src/descriptor.rs b/src/descriptor.rs index 3e76bee..8f3dd2f 100644 --- a/src/descriptor.rs +++ b/src/descriptor.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/lib.rs b/src/lib.rs index da5a756..8eee9d3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,8 @@ pub use crate::cached_size::CachedSize; #[cfg(feature = "bytes")] pub use crate::chars::Chars; pub use crate::clear::Clear; +pub use crate::coded_input_stream::CodedInputStream; +pub use crate::coded_output_stream::CodedOutputStream; pub use crate::enums::ProtobufEnum; pub use crate::error::ProtobufError; pub use crate::error::ProtobufResult; @@ -34,9 +36,6 @@ pub use crate::message::Message; pub use crate::repeated::RepeatedField; pub use crate::singular::SingularField; pub use crate::singular::SingularPtrField; -pub use crate::stream::wire_format; -pub use crate::stream::CodedInputStream; -pub use crate::stream::CodedOutputStream; pub use crate::unknown::UnknownFields; pub use crate::unknown::UnknownFieldsIter; pub use crate::unknown::UnknownValue; @@ -49,7 +48,11 @@ pub mod descriptor; pub mod plugin; pub mod rustproto; +pub mod wire_format; + mod clear; +mod coded_input_stream; +mod coded_output_stream; pub mod compiler_plugin; mod enums; pub mod error; @@ -62,7 +65,6 @@ pub mod reflect; mod repeated; pub mod rt; mod singular; -pub mod stream; pub mod text_format; pub mod types; pub mod well_known_types; diff --git a/src/message.rs b/src/message.rs index b22ace2..e9d6b30 100644 --- a/src/message.rs +++ b/src/message.rs @@ -8,14 +8,14 @@ use std::io::Write; use bytes::Bytes; use crate::clear::Clear; +use crate::coded_input_stream::CodedInputStream; +use crate::coded_input_stream::WithCodedInputStream; +use crate::coded_output_stream::with_coded_output_stream_to_bytes; +use crate::coded_output_stream::CodedOutputStream; +use crate::coded_output_stream::WithCodedOutputStream; use crate::error::ProtobufError; use crate::error::ProtobufResult; use crate::reflect::MessageDescriptor; -use crate::stream::with_coded_output_stream_to_bytes; -use crate::stream::CodedInputStream; -use crate::stream::CodedOutputStream; -use crate::stream::WithCodedInputStream; -use crate::stream::WithCodedOutputStream; use crate::unknown::UnknownFields; /// Trait implemented for all generated structs for protobuf messages. diff --git a/src/plugin.rs b/src/plugin.rs index 007378f..77e1666 100644 --- a/src/plugin.rs +++ b/src/plugin.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/rt.rs b/src/rt.rs index 7af7c2c..3bb0280 100644 --- a/src/rt.rs +++ b/src/rt.rs @@ -10,6 +10,8 @@ use crate::chars::Chars; #[cfg(feature = "bytes")] use bytes::Bytes; +use crate::coded_input_stream::CodedInputStream; +use crate::coded_output_stream::CodedOutputStream; use crate::enums::ProtobufEnum; use crate::error::ProtobufError; use crate::error::ProtobufResult; @@ -18,19 +20,13 @@ use crate::message::*; use crate::repeated::RepeatedField; use crate::singular::SingularField; use crate::singular::SingularPtrField; -use crate::stream::wire_format; -use crate::stream::wire_format::WireType; -use crate::stream::wire_format::WireTypeFixed32; -use crate::stream::wire_format::WireTypeFixed64; -use crate::stream::wire_format::WireTypeLengthDelimited; -use crate::stream::wire_format::WireTypeVarint; -use crate::stream::CodedInputStream; -use crate::stream::CodedOutputStream; use crate::types::*; use crate::zigzag::*; pub use crate::lazy_v2::LazyV2; use crate::unknown::UnknownFields; +use crate::wire_format; +use crate::wire_format::WireType; /// Given `u64` value compute varint encoded length. pub fn compute_raw_varint64_size(value: u64) -> u32 { @@ -184,16 +180,16 @@ pub fn vec_packed_enum_size(field_number: u32, vec: &[E]) -> u3 /// Compute tag size. Size of tag does not depend on wire type. pub fn tag_size(field_number: u32) -> u32 { - wire_format::Tag::make(field_number, WireTypeFixed64) + wire_format::Tag::make(field_number, WireType::WireTypeFixed64) .value() .len_varint() } fn value_size_no_tag(value: T, wt: WireType) -> u32 { match wt { - WireTypeFixed64 => 8, - WireTypeFixed32 => 4, - WireTypeVarint => value.len_varint(), + WireType::WireTypeFixed64 => 8, + WireType::WireTypeFixed32 => 4, + WireType::WireTypeVarint => value.len_varint(), _ => panic!(), } } @@ -267,8 +263,8 @@ pub fn read_repeated_int32_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_int32_into(target), - WireTypeVarint => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_int32_into(target), + WireType::WireTypeVarint => { target.push(is.read_int32()?); Ok(()) } @@ -283,8 +279,8 @@ pub fn read_repeated_int64_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_int64_into(target), - WireTypeVarint => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_int64_into(target), + WireType::WireTypeVarint => { target.push(is.read_int64()?); Ok(()) } @@ -299,8 +295,8 @@ pub fn read_repeated_uint32_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_uint32_into(target), - WireTypeVarint => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_uint32_into(target), + WireType::WireTypeVarint => { target.push(is.read_uint32()?); Ok(()) } @@ -315,8 +311,8 @@ pub fn read_repeated_uint64_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_uint64_into(target), - WireTypeVarint => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_uint64_into(target), + WireType::WireTypeVarint => { target.push(is.read_uint64()?); Ok(()) } @@ -331,8 +327,8 @@ pub fn read_repeated_sint32_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_sint32_into(target), - WireTypeVarint => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_sint32_into(target), + WireType::WireTypeVarint => { target.push(is.read_sint32()?); Ok(()) } @@ -347,8 +343,8 @@ pub fn read_repeated_sint64_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_sint64_into(target), - WireTypeVarint => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_sint64_into(target), + WireType::WireTypeVarint => { target.push(is.read_sint64()?); Ok(()) } @@ -363,8 +359,8 @@ pub fn read_repeated_fixed32_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_fixed32_into(target), - WireTypeFixed32 => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_fixed32_into(target), + WireType::WireTypeFixed32 => { target.push(is.read_fixed32()?); Ok(()) } @@ -379,8 +375,8 @@ pub fn read_repeated_fixed64_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_fixed64_into(target), - WireTypeFixed64 => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_fixed64_into(target), + WireType::WireTypeFixed64 => { target.push(is.read_fixed64()?); Ok(()) } @@ -395,8 +391,8 @@ pub fn read_repeated_sfixed32_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_sfixed32_into(target), - WireTypeFixed32 => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_sfixed32_into(target), + WireType::WireTypeFixed32 => { target.push(is.read_sfixed32()?); Ok(()) } @@ -411,8 +407,8 @@ pub fn read_repeated_sfixed64_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_sfixed64_into(target), - WireTypeFixed64 => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_sfixed64_into(target), + WireType::WireTypeFixed64 => { target.push(is.read_sfixed64()?); Ok(()) } @@ -427,8 +423,8 @@ pub fn read_repeated_double_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_double_into(target), - WireTypeFixed64 => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_double_into(target), + WireType::WireTypeFixed64 => { target.push(is.read_double()?); Ok(()) } @@ -443,8 +439,8 @@ pub fn read_repeated_float_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_float_into(target), - WireTypeFixed32 => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_float_into(target), + WireType::WireTypeFixed32 => { target.push(is.read_float()?); Ok(()) } @@ -459,8 +455,8 @@ pub fn read_repeated_bool_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_bool_into(target), - WireTypeVarint => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_bool_into(target), + WireType::WireTypeVarint => { target.push(is.read_bool()?); Ok(()) } @@ -476,8 +472,8 @@ pub fn read_repeated_enum_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_repeated_packed_enum_into(target), - WireTypeVarint => { + WireType::WireTypeLengthDelimited => is.read_repeated_packed_enum_into(target), + WireType::WireTypeVarint => { target.push(is.read_enum()?); Ok(()) } @@ -533,13 +529,13 @@ pub fn read_repeated_enum_with_unknown_fields_into( unknown_fields: &mut UnknownFields, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => read_repeated_packed_enum_with_unknown_fields_into( + WireType::WireTypeLengthDelimited => read_repeated_packed_enum_with_unknown_fields_into( is, target, field_number, unknown_fields, ), - WireTypeVarint => { + WireType::WireTypeVarint => { read_enum_with_unknown_fields_into(is, |e| target.push(e), field_number, unknown_fields) } _ => Err(unexpected_wire_type(wire_type)), @@ -593,7 +589,7 @@ pub fn read_repeated_string_into( target: &mut RepeatedField, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { let tmp = target.push_default(); is.read_string_into(tmp) } @@ -609,7 +605,7 @@ pub fn read_repeated_carllerche_string_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { target.push(is.read_carllerche_chars()?); Ok(()) } @@ -624,7 +620,7 @@ pub fn read_singular_string_into( target: &mut SingularField, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { let tmp = target.set_default(); is.read_string_into(tmp) } @@ -640,7 +636,7 @@ pub fn read_singular_carllerche_string_into( target: &mut Option, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { *target = Some(is.read_carllerche_chars()?); Ok(()) } @@ -655,7 +651,7 @@ pub fn read_singular_proto3_string_into( target: &mut String, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_string_into(target), + WireType::WireTypeLengthDelimited => is.read_string_into(target), _ => Err(unexpected_wire_type(wire_type)), } } @@ -668,7 +664,7 @@ pub fn read_singular_proto3_carllerche_string_into( target: &mut Chars, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { *target = is.read_carllerche_chars()?; Ok(()) } @@ -683,7 +679,7 @@ pub fn read_repeated_bytes_into( target: &mut RepeatedField>, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { let tmp = target.push_default(); is.read_bytes_into(tmp) } @@ -699,7 +695,7 @@ pub fn read_repeated_carllerche_bytes_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { target.push(is.read_carllerche_bytes()?); Ok(()) } @@ -714,7 +710,7 @@ pub fn read_singular_bytes_into( target: &mut SingularField>, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { let tmp = target.set_default(); is.read_bytes_into(tmp) } @@ -730,7 +726,7 @@ pub fn read_singular_carllerche_bytes_into( target: &mut Option, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { *target = Some(is.read_carllerche_bytes()?); Ok(()) } @@ -745,7 +741,7 @@ pub fn read_singular_proto3_bytes_into( target: &mut Vec, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => is.read_bytes_into(target), + WireType::WireTypeLengthDelimited => is.read_bytes_into(target), _ => Err(unexpected_wire_type(wire_type)), } } @@ -758,7 +754,7 @@ pub fn read_singular_proto3_carllerche_bytes_into( target: &mut Bytes, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { *target = is.read_carllerche_bytes()?; Ok(()) } @@ -773,7 +769,7 @@ pub fn read_repeated_message_into( target: &mut RepeatedField, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { is.incr_recursion()?; let tmp = target.push_default(); let res = is.merge_message(tmp); @@ -791,7 +787,7 @@ pub fn read_singular_message_into( target: &mut SingularPtrField, ) -> ProtobufResult<()> { match wire_type { - WireTypeLengthDelimited => { + WireType::WireTypeLengthDelimited => { is.incr_recursion()?; let tmp = target.set_default(); let res = is.merge_message(tmp); diff --git a/src/rustproto.rs b/src/rustproto.rs index d4284e7..83854e4 100644 --- a/src/rustproto.rs +++ b/src/rustproto.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/stream.rs b/src/stream.rs deleted file mode 100644 index 5ec7df2..0000000 --- a/src/stream.rs +++ /dev/null @@ -1,1803 +0,0 @@ -#![doc(hidden)] - -//! `CodedInputStream` and `CodedOutputStream` implementations - -use std::io; -use std::io::BufRead; -use std::io::Read; -use std::io::Write; -use std::mem; -use std::slice; - -#[cfg(feature = "bytes")] -use crate::chars::Chars; -#[cfg(feature = "bytes")] -use bytes::Bytes; - -use crate::buf_read_iter::BufReadIter; -use crate::enums::ProtobufEnum; -use crate::error::ProtobufError; -use crate::error::ProtobufResult; -use crate::error::WireError; -use crate::message::Message; -use crate::misc::remaining_capacity_as_slice_mut; -use crate::misc::remove_lifetime_mut; -use crate::unknown::UnknownFields; -use crate::unknown::UnknownValue; -use crate::unknown::UnknownValueRef; -use crate::varint; -use crate::zigzag::decode_zig_zag_32; -use crate::zigzag::decode_zig_zag_64; -use crate::zigzag::encode_zig_zag_32; -use crate::zigzag::encode_zig_zag_64; - -/// Equal to the default buffer size of `BufWriter`, so when -/// `CodedOutputStream` wraps `BufWriter`, it often skips double buffering. -const OUTPUT_STREAM_BUFFER_SIZE: usize = 8 * 1024; - -/// Default recursion level limit. 100 is the default value of C++'s implementation. -const DEFAULT_RECURSION_LIMIT: u32 = 100; - -/// Max allocated vec when reading length-delimited from unknown input stream -pub(crate) const READ_RAW_BYTES_MAX_ALLOC: usize = 10_000_000; - -/// Serialization constants. -pub mod wire_format { - // TODO: temporary - pub use self::WireType::*; - - /// Tag occupies 3 bits - pub const TAG_TYPE_BITS: u32 = 3; - /// Tag mask - pub const TAG_TYPE_MASK: u32 = (1u32 << TAG_TYPE_BITS) - 1; - /// Max possible field number - pub const FIELD_NUMBER_MAX: u32 = 0x1fffffff; - - /// One of six defined protobuf wire types - #[derive(PartialEq, Eq, Clone, Debug)] - pub enum WireType { - /// Varint (e. g. `int32` or `sint64`) - WireTypeVarint = 0, - /// Fixed size 64 bit (e. g. `fixed64` or `double`) - WireTypeFixed64 = 1, - /// Length-delimited (e. g. `message` or `string`) - WireTypeLengthDelimited = 2, - /// Groups are not supported by rust-protobuf - WireTypeStartGroup = 3, - /// Groups are not supported by rust-protobuf - WireTypeEndGroup = 4, - /// Fixed size 64 bit (e. g. `fixed32` or `float`) - WireTypeFixed32 = 5, - } - - impl Copy for WireType {} - - impl WireType { - /// Parse wire type - pub fn new(n: u32) -> Option { - match n { - 0 => Some(WireTypeVarint), - 1 => Some(WireTypeFixed64), - 2 => Some(WireTypeLengthDelimited), - 3 => Some(WireTypeStartGroup), - 4 => Some(WireTypeEndGroup), - 5 => Some(WireTypeFixed32), - _ => None, - } - } - } - - /// Parsed protobuf tag, which is a pair of field number and wire type - #[derive(Clone)] - pub struct Tag { - field_number: u32, - wire_type: WireType, - } - - impl Copy for Tag {} - - impl Tag { - /// Pack a tag to integer - pub fn value(self) -> u32 { - (self.field_number << TAG_TYPE_BITS) | (self.wire_type as u32) - } - - /// Parse integer into `Tag` object - // TODO: should return Result instead of Option - pub fn new(value: u32) -> Option { - let wire_type = WireType::new(value & TAG_TYPE_MASK); - if wire_type.is_none() { - return None; - } - let field_number = value >> TAG_TYPE_BITS; - if field_number == 0 { - return None; - } - Some(Tag { - field_number: field_number, - wire_type: wire_type.unwrap(), - }) - } - - /// Create a tag from a field number and wire type. - /// - /// # Panics - /// - /// If field number is outside of allowed range. - pub fn make(field_number: u32, wire_type: WireType) -> Tag { - assert!(field_number > 0 && field_number <= FIELD_NUMBER_MAX); - Tag { - field_number: field_number, - wire_type: wire_type, - } - } - - /// Tag as pair of (field number, wire type) - pub fn unpack(self) -> (u32, WireType) { - (self.field_number(), self.wire_type()) - } - - fn wire_type(self) -> WireType { - self.wire_type - } - - /// Protobuf field number - pub fn field_number(self) -> u32 { - self.field_number - } - } -} - -/// Buffered read with handy utilities. -pub struct CodedInputStream<'a> { - source: BufReadIter<'a>, - recursion_level: u32, - recursion_limit: u32, -} - -impl<'a> CodedInputStream<'a> { - /// Wrap a `Read`. - /// - /// Note resulting `CodedInputStream` is buffered even if `Read` is not. - pub fn new(read: &'a mut dyn Read) -> CodedInputStream<'a> { - CodedInputStream::from_buf_read_iter(BufReadIter::from_read(read)) - } - - /// Create from `BufRead`. - /// - /// `CodedInputStream` will utilize `BufRead` buffer. - pub fn from_buffered_reader(buf_read: &'a mut dyn BufRead) -> CodedInputStream<'a> { - CodedInputStream::from_buf_read_iter(BufReadIter::from_buf_read(buf_read)) - } - - /// Read from byte slice - pub fn from_bytes(bytes: &'a [u8]) -> CodedInputStream<'a> { - CodedInputStream::from_buf_read_iter(BufReadIter::from_byte_slice(bytes)) - } - - /// Read from `Bytes`. - /// - /// `CodedInputStream` operations like - /// [`read_carllerche_bytes`](crate::CodedInputStream::read_carllerche_bytes) - /// will return a shared copy of this bytes object. - #[cfg(feature = "bytes")] - pub fn from_carllerche_bytes(bytes: &'a Bytes) -> CodedInputStream<'a> { - CodedInputStream::from_buf_read_iter(BufReadIter::from_bytes(bytes)) - } - - fn from_buf_read_iter(source: BufReadIter<'a>) -> CodedInputStream<'a> { - CodedInputStream { - source: source, - recursion_level: 0, - recursion_limit: DEFAULT_RECURSION_LIMIT, - } - } - - /// Set the recursion limit. - pub fn set_recursion_limit(&mut self, limit: u32) { - self.recursion_limit = limit; - } - - #[inline] - pub(crate) fn incr_recursion(&mut self) -> ProtobufResult<()> { - if self.recursion_level >= self.recursion_limit { - return Err(ProtobufError::WireError(WireError::OverRecursionLimit)); - } - self.recursion_level += 1; - Ok(()) - } - - #[inline] - pub(crate) fn decr_recursion(&mut self) { - self.recursion_level -= 1; - } - - /// How many bytes processed - pub fn pos(&self) -> u64 { - self.source.pos() - } - - /// How many bytes until current limit - pub fn bytes_until_limit(&self) -> u64 { - self.source.bytes_until_limit() - } - - /// Read bytes into given `buf`. - /// - /// Return `0` on EOF. - // TODO: overload with `Read::read` - pub fn read(&mut self, buf: &mut [u8]) -> ProtobufResult<()> { - self.source.read_exact(buf)?; - Ok(()) - } - - /// Read exact number of bytes as `Bytes` object. - /// - /// This operation returns a shared view if `CodedInputStream` is - /// constructed with `Bytes` parameter. - #[cfg(feature = "bytes")] - fn read_raw_callerche_bytes(&mut self, count: usize) -> ProtobufResult { - self.source.read_exact_bytes(count) - } - - /// Read one byte - #[inline(always)] - pub fn read_raw_byte(&mut self) -> ProtobufResult { - self.source.read_byte() - } - - /// Push new limit, return previous limit. - pub fn push_limit(&mut self, limit: u64) -> ProtobufResult { - self.source.push_limit(limit) - } - - /// Restore previous limit. - pub fn pop_limit(&mut self, old_limit: u64) { - self.source.pop_limit(old_limit); - } - - /// Are we at EOF? - #[inline(always)] - pub fn eof(&mut self) -> ProtobufResult { - self.source.eof() - } - - /// Check we are at EOF. - /// - /// Return error if we are not at EOF. - pub fn check_eof(&mut self) -> ProtobufResult<()> { - let eof = self.eof()?; - if !eof { - return Err(ProtobufError::WireError(WireError::UnexpectedEof)); - } - Ok(()) - } - - fn read_raw_varint64_slow(&mut self) -> ProtobufResult { - let mut r: u64 = 0; - let mut i = 0; - loop { - if i == 10 { - return Err(ProtobufError::WireError(WireError::IncorrectVarint)); - } - let b = self.read_raw_byte()?; - // TODO: may overflow if i == 9 - r = r | (((b & 0x7f) as u64) << (i * 7)); - i += 1; - if b < 0x80 { - return Ok(r); - } - } - } - - /// Read varint - #[inline(always)] - pub fn read_raw_varint64(&mut self) -> ProtobufResult { - 'slow: loop { - let ret; - let consume; - - loop { - let rem = self.source.remaining_in_buf(); - - if rem.len() >= 1 { - // most varints are in practice fit in 1 byte - if rem[0] < 0x80 { - ret = rem[0] as u64; - consume = 1; - } else { - // handle case of two bytes too - if rem.len() >= 2 && rem[1] < 0x80 { - ret = (rem[0] & 0x7f) as u64 | (rem[1] as u64) << 7; - consume = 2; - } else if rem.len() >= 10 { - // Read from array when buf at at least 10 bytes, - // max len for varint. - let mut r: u64 = 0; - let mut i: usize = 0; - { - let rem = rem; - loop { - if i == 10 { - return Err(ProtobufError::WireError( - WireError::IncorrectVarint, - )); - } - - let b = if true { - // skip range check - unsafe { *rem.get_unchecked(i) } - } else { - rem[i] - }; - - // TODO: may overflow if i == 9 - r = r | (((b & 0x7f) as u64) << (i * 7)); - i += 1; - if b < 0x80 { - break; - } - } - } - consume = i; - ret = r; - } else { - break 'slow; - } - } - } else { - break 'slow; - } - break; - } - - self.source.consume(consume); - return Ok(ret); - } - - self.read_raw_varint64_slow() - } - - /// Read varint - #[inline(always)] - pub fn read_raw_varint32(&mut self) -> ProtobufResult { - self.read_raw_varint64().map(|v| v as u32) - } - - /// Read little-endian 32-bit integer - pub fn read_raw_little_endian32(&mut self) -> ProtobufResult { - let mut r = 0u32; - let bytes: &mut [u8] = unsafe { - let p: *mut u8 = mem::transmute(&mut r); - slice::from_raw_parts_mut(p, mem::size_of::()) - }; - self.read(bytes)?; - Ok(r.to_le()) - } - - /// Read little-endian 64-bit integer - pub fn read_raw_little_endian64(&mut self) -> ProtobufResult { - let mut r = 0u64; - let bytes: &mut [u8] = unsafe { - let p: *mut u8 = mem::transmute(&mut r); - slice::from_raw_parts_mut(p, mem::size_of::()) - }; - self.read(bytes)?; - Ok(r.to_le()) - } - - /// Read tag - #[inline] - pub fn read_tag(&mut self) -> ProtobufResult { - let v = self.read_raw_varint32()?; - match wire_format::Tag::new(v) { - Some(tag) => Ok(tag), - None => Err(ProtobufError::WireError(WireError::IncorrectTag(v))), - } - } - - /// Read tag, return it is pair (field number, wire type) - #[inline] - pub fn read_tag_unpack(&mut self) -> ProtobufResult<(u32, wire_format::WireType)> { - self.read_tag().map(|t| t.unpack()) - } - - /// Read `double` - pub fn read_double(&mut self) -> ProtobufResult { - let bits = self.read_raw_little_endian64()?; - unsafe { Ok(mem::transmute::(bits)) } - } - - /// Read `float` - pub fn read_float(&mut self) -> ProtobufResult { - let bits = self.read_raw_little_endian32()?; - unsafe { Ok(mem::transmute::(bits)) } - } - - /// Read `int64` - pub fn read_int64(&mut self) -> ProtobufResult { - self.read_raw_varint64().map(|v| v as i64) - } - - /// Read `int32` - pub fn read_int32(&mut self) -> ProtobufResult { - self.read_raw_varint32().map(|v| v as i32) - } - - /// Read `uint64` - pub fn read_uint64(&mut self) -> ProtobufResult { - self.read_raw_varint64() - } - - /// Read `uint32` - pub fn read_uint32(&mut self) -> ProtobufResult { - self.read_raw_varint32() - } - - /// Read `sint64` - pub fn read_sint64(&mut self) -> ProtobufResult { - self.read_uint64().map(decode_zig_zag_64) - } - - /// Read `sint32` - pub fn read_sint32(&mut self) -> ProtobufResult { - self.read_uint32().map(decode_zig_zag_32) - } - - /// Read `fixed64` - pub fn read_fixed64(&mut self) -> ProtobufResult { - self.read_raw_little_endian64() - } - - /// Read `fixed32` - pub fn read_fixed32(&mut self) -> ProtobufResult { - self.read_raw_little_endian32() - } - - /// Read `sfixed64` - pub fn read_sfixed64(&mut self) -> ProtobufResult { - self.read_raw_little_endian64().map(|v| v as i64) - } - - /// Read `sfixed32` - pub fn read_sfixed32(&mut self) -> ProtobufResult { - self.read_raw_little_endian32().map(|v| v as i32) - } - - /// Read `bool` - pub fn read_bool(&mut self) -> ProtobufResult { - self.read_raw_varint32().map(|v| v != 0) - } - - /// Read `enum` as `ProtobufEnum` - pub fn read_enum(&mut self) -> ProtobufResult { - let i = self.read_int32()?; - match ProtobufEnum::from_i32(i) { - Some(e) => Ok(e), - None => Err(ProtobufError::WireError(WireError::InvalidEnumValue(i))), - } - } - - /// Read `repeated` packed `double` - pub fn read_repeated_packed_double_into( - &mut self, - target: &mut Vec, - ) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - - target.reserve((len / 4) as usize); - - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_double()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read `repeated` packed `float` - pub fn read_repeated_packed_float_into(&mut self, target: &mut Vec) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - - target.reserve((len / 4) as usize); - - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_float()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read `repeated` packed `int64` - pub fn read_repeated_packed_int64_into(&mut self, target: &mut Vec) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - let old_limit = self.push_limit(len as u64)?; - while !self.eof()? { - target.push(self.read_int64()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read repeated packed `int32` - pub fn read_repeated_packed_int32_into(&mut self, target: &mut Vec) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_int32()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read repeated packed `uint64` - pub fn read_repeated_packed_uint64_into( - &mut self, - target: &mut Vec, - ) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_uint64()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read repeated packed `uint32` - pub fn read_repeated_packed_uint32_into( - &mut self, - target: &mut Vec, - ) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_uint32()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read repeated packed `sint64` - pub fn read_repeated_packed_sint64_into( - &mut self, - target: &mut Vec, - ) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_sint64()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read repeated packed `sint32` - pub fn read_repeated_packed_sint32_into( - &mut self, - target: &mut Vec, - ) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_sint32()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read repeated packed `fixed64` - pub fn read_repeated_packed_fixed64_into( - &mut self, - target: &mut Vec, - ) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - - target.reserve((len / 8) as usize); - - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_fixed64()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read repeated packed `fixed32` - pub fn read_repeated_packed_fixed32_into( - &mut self, - target: &mut Vec, - ) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - - target.reserve((len / 4) as usize); - - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_fixed32()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read repeated packed `sfixed64` - pub fn read_repeated_packed_sfixed64_into( - &mut self, - target: &mut Vec, - ) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - - target.reserve((len / 8) as usize); - - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_sfixed64()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read repeated packed `sfixed32` - pub fn read_repeated_packed_sfixed32_into( - &mut self, - target: &mut Vec, - ) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - - target.reserve((len / 4) as usize); - - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_sfixed32()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read repeated packed `bool` - pub fn read_repeated_packed_bool_into(&mut self, target: &mut Vec) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - - // regular bool value is 1-byte size - target.reserve(len as usize); - - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_bool()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read repeated packed `enum` into `ProtobufEnum` - pub fn read_repeated_packed_enum_into( - &mut self, - target: &mut Vec, - ) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - let old_limit = self.push_limit(len)?; - while !self.eof()? { - target.push(self.read_enum()?); - } - self.pop_limit(old_limit); - Ok(()) - } - - /// Read `UnknownValue` - pub fn read_unknown( - &mut self, - wire_type: wire_format::WireType, - ) -> ProtobufResult { - match wire_type { - wire_format::WireTypeVarint => { - self.read_raw_varint64().map(|v| UnknownValue::Varint(v)) - } - wire_format::WireTypeFixed64 => self.read_fixed64().map(|v| UnknownValue::Fixed64(v)), - wire_format::WireTypeFixed32 => self.read_fixed32().map(|v| UnknownValue::Fixed32(v)), - wire_format::WireTypeLengthDelimited => { - let len = self.read_raw_varint32()?; - self.read_raw_bytes(len) - .map(|v| UnknownValue::LengthDelimited(v)) - } - _ => Err(ProtobufError::WireError(WireError::UnexpectedWireType( - wire_type, - ))), - } - } - - /// Skip field - pub fn skip_field(&mut self, wire_type: wire_format::WireType) -> ProtobufResult<()> { - self.read_unknown(wire_type).map(|_| ()) - } - - /// Read raw bytes into the supplied vector. The vector will be resized as needed and - /// overwritten. - pub fn read_raw_bytes_into(&mut self, count: u32, target: &mut Vec) -> ProtobufResult<()> { - if false { - // Master uses this version, but keep existing version for a while - // to avoid possible breakages. - return self.source.read_exact_to_vec(count as usize, target); - } - - let count = count as usize; - - // TODO: also do some limits when reading from unlimited source - if count as u64 > self.source.bytes_until_limit() { - return Err(ProtobufError::WireError(WireError::TruncatedMessage)); - } - - unsafe { - target.set_len(0); - } - - if count >= READ_RAW_BYTES_MAX_ALLOC { - // avoid calling `reserve` on buf with very large buffer: could be a malformed message - - let mut take = self.by_ref().take(count as u64); - take.read_to_end(target)?; - - if target.len() != count { - return Err(ProtobufError::WireError(WireError::TruncatedMessage)); - } - } else { - target.reserve(count); - unsafe { - target.set_len(count); - } - - self.source.read_exact(target)?; - } - Ok(()) - } - - /// Read exact number of bytes - pub fn read_raw_bytes(&mut self, count: u32) -> ProtobufResult> { - let mut r = Vec::new(); - self.read_raw_bytes_into(count, &mut r)?; - Ok(r) - } - - /// Skip exact number of bytes - pub fn skip_raw_bytes(&mut self, count: u32) -> ProtobufResult<()> { - // TODO: make it more efficient - self.read_raw_bytes(count).map(|_| ()) - } - - /// Read `bytes` field, length delimited - pub fn read_bytes(&mut self) -> ProtobufResult> { - let mut r = Vec::new(); - self.read_bytes_into(&mut r)?; - Ok(r) - } - - /// Read `bytes` field, length delimited - #[cfg(feature = "bytes")] - pub fn read_carllerche_bytes(&mut self) -> ProtobufResult { - let len = self.read_raw_varint32()?; - self.read_raw_callerche_bytes(len as usize) - } - - /// Read `string` field, length delimited - #[cfg(feature = "bytes")] - pub fn read_carllerche_chars(&mut self) -> ProtobufResult { - let bytes = self.read_carllerche_bytes()?; - Ok(Chars::from_bytes(bytes)?) - } - - /// Read `bytes` field, length delimited - pub fn read_bytes_into(&mut self, target: &mut Vec) -> ProtobufResult<()> { - let len = self.read_raw_varint32()?; - self.read_raw_bytes_into(len, target)?; - Ok(()) - } - - /// Read `string` field, length delimited - pub fn read_string(&mut self) -> ProtobufResult { - let mut r = String::new(); - self.read_string_into(&mut r)?; - Ok(r) - } - - /// Read `string` field, length delimited - pub fn read_string_into(&mut self, target: &mut String) -> ProtobufResult<()> { - target.clear(); - // take target's buffer - let mut vec = mem::replace(target, String::new()).into_bytes(); - self.read_bytes_into(&mut vec)?; - - let s = match String::from_utf8(vec) { - Ok(t) => t, - Err(_) => return Err(ProtobufError::WireError(WireError::Utf8Error)), - }; - *target = s; - Ok(()) - } - - /// Read message, do not check if message is initialized - pub fn merge_message(&mut self, message: &mut M) -> ProtobufResult<()> { - let len = self.read_raw_varint64()?; - let old_limit = self.push_limit(len)?; - message.merge_from(self)?; - self.pop_limit(old_limit); - Ok(()) - } - - /// Read message - pub fn read_message(&mut self) -> ProtobufResult { - let mut r: M = Message::new(); - self.merge_message(&mut r)?; - r.check_initialized()?; - Ok(r) - } -} - -impl<'a> Read for CodedInputStream<'a> { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - self.source.read(buf).map_err(Into::into) - } -} - -impl<'a> BufRead for CodedInputStream<'a> { - fn fill_buf(&mut self) -> io::Result<&[u8]> { - self.source.fill_buf().map_err(Into::into) - } - - fn consume(&mut self, amt: usize) { - self.source.consume(amt) - } -} - -#[doc(hidden)] -pub trait WithCodedOutputStream { - fn with_coded_output_stream(self, cb: F) -> ProtobufResult - where - F: FnOnce(&mut CodedOutputStream) -> ProtobufResult; -} - -impl<'a> WithCodedOutputStream for &'a mut (dyn Write + 'a) { - fn with_coded_output_stream(self, cb: F) -> ProtobufResult - where - F: FnOnce(&mut CodedOutputStream) -> ProtobufResult, - { - let mut os = CodedOutputStream::new(self); - let r = cb(&mut os)?; - os.flush()?; - Ok(r) - } -} - -impl<'a> WithCodedOutputStream for &'a mut Vec { - fn with_coded_output_stream(mut self, cb: F) -> ProtobufResult - where - F: FnOnce(&mut CodedOutputStream) -> ProtobufResult, - { - let mut os = CodedOutputStream::vec(&mut self); - let r = cb(&mut os)?; - os.flush()?; - Ok(r) - } -} - -#[doc(hidden)] -pub fn with_coded_output_stream_to_bytes(cb: F) -> ProtobufResult> -where - F: FnOnce(&mut CodedOutputStream) -> ProtobufResult<()>, -{ - let mut v = Vec::new(); - v.with_coded_output_stream(cb)?; - Ok(v) -} - -/// Helper internal utility, should not be used directly -#[doc(hidden)] -pub trait WithCodedInputStream { - fn with_coded_input_stream(self, cb: F) -> ProtobufResult - where - F: FnOnce(&mut CodedInputStream) -> ProtobufResult; -} - -impl<'a> WithCodedInputStream for &'a mut (dyn Read + 'a) { - fn with_coded_input_stream(self, cb: F) -> ProtobufResult - where - F: FnOnce(&mut CodedInputStream) -> ProtobufResult, - { - let mut is = CodedInputStream::new(self); - let r = cb(&mut is)?; - is.check_eof()?; - Ok(r) - } -} - -impl<'a> WithCodedInputStream for &'a mut (dyn BufRead + 'a) { - fn with_coded_input_stream(self, cb: F) -> ProtobufResult - where - F: FnOnce(&mut CodedInputStream) -> ProtobufResult, - { - let mut is = CodedInputStream::from_buffered_reader(self); - let r = cb(&mut is)?; - is.check_eof()?; - Ok(r) - } -} - -impl<'a> WithCodedInputStream for &'a [u8] { - fn with_coded_input_stream(self, cb: F) -> ProtobufResult - where - F: FnOnce(&mut CodedInputStream) -> ProtobufResult, - { - let mut is = CodedInputStream::from_bytes(self); - let r = cb(&mut is)?; - is.check_eof()?; - Ok(r) - } -} - -#[cfg(feature = "bytes")] -impl<'a> WithCodedInputStream for &'a Bytes { - fn with_coded_input_stream(self, cb: F) -> ProtobufResult - where - F: FnOnce(&mut CodedInputStream) -> ProtobufResult, - { - let mut is = CodedInputStream::from_carllerche_bytes(self); - let r = cb(&mut is)?; - is.check_eof()?; - Ok(r) - } -} - -enum OutputTarget<'a> { - Write(&'a mut dyn Write, Vec), - Vec(&'a mut Vec), - Bytes, -} - -/// Buffered write with handy utilities -pub struct CodedOutputStream<'a> { - target: OutputTarget<'a>, - // alias to buf from target - buffer: &'a mut [u8], - // within buffer - position: usize, -} - -impl<'a> CodedOutputStream<'a> { - /// Construct from given `Write`. - /// - /// `CodedOutputStream` is buffered even if `Write` is not - pub fn new(writer: &'a mut dyn Write) -> CodedOutputStream<'a> { - let buffer_len = OUTPUT_STREAM_BUFFER_SIZE; - - let mut buffer_storage = Vec::with_capacity(buffer_len); - unsafe { - buffer_storage.set_len(buffer_len); - } - - let buffer = unsafe { remove_lifetime_mut(&mut buffer_storage as &mut [u8]) }; - - CodedOutputStream { - target: OutputTarget::Write(writer, buffer_storage), - buffer: buffer, - position: 0, - } - } - - /// `CodedOutputStream` which writes directly to bytes. - /// - /// Attempt to write more than bytes capacity results in error. - pub fn bytes(bytes: &'a mut [u8]) -> CodedOutputStream<'a> { - CodedOutputStream { - target: OutputTarget::Bytes, - buffer: bytes, - position: 0, - } - } - - /// `CodedOutputStream` which writes directly to `Vec`. - /// - /// Caller should call `flush` at the end to guarantee vec contains - /// all written data. - pub fn vec(vec: &'a mut Vec) -> CodedOutputStream<'a> { - CodedOutputStream { - target: OutputTarget::Vec(vec), - buffer: &mut [], - position: 0, - } - } - - /// Check if EOF is reached. - /// - /// # Panics - /// - /// If underlying write has no EOF - pub fn check_eof(&self) { - match self.target { - OutputTarget::Bytes => { - assert_eq!(self.buffer.len() as u64, self.position as u64); - } - OutputTarget::Write(..) | OutputTarget::Vec(..) => { - panic!("must not be called with Writer or Vec"); - } - } - } - - fn refresh_buffer(&mut self) -> ProtobufResult<()> { - match self.target { - OutputTarget::Write(ref mut write, _) => { - write.write_all(&self.buffer[0..self.position as usize])?; - self.position = 0; - } - OutputTarget::Vec(ref mut vec) => unsafe { - let vec_len = vec.len(); - assert!(vec_len + self.position <= vec.capacity()); - vec.set_len(vec_len + self.position); - vec.reserve(1); - self.buffer = remove_lifetime_mut(remaining_capacity_as_slice_mut(vec)); - self.position = 0; - }, - OutputTarget::Bytes => { - panic!("refresh_buffer must not be called on CodedOutputStream create from slice"); - } - } - Ok(()) - } - - /// Flush the buffer to underlying write - pub fn flush(&mut self) -> ProtobufResult<()> { - match self.target { - OutputTarget::Bytes => Ok(()), - OutputTarget::Write(..) | OutputTarget::Vec(..) => { - // TODO: must not reserve additional in Vec - self.refresh_buffer() - } - } - } - - /// Write a byte - pub fn write_raw_byte(&mut self, byte: u8) -> ProtobufResult<()> { - if self.position as usize == self.buffer.len() { - self.refresh_buffer()?; - } - self.buffer[self.position as usize] = byte; - self.position += 1; - Ok(()) - } - - /// Write bytes - pub fn write_raw_bytes(&mut self, bytes: &[u8]) -> ProtobufResult<()> { - if bytes.len() <= self.buffer.len() - self.position { - let bottom = self.position as usize; - let top = bottom + (bytes.len() as usize); - self.buffer[bottom..top].copy_from_slice(bytes); - self.position += bytes.len(); - return Ok(()); - } - - self.refresh_buffer()?; - - assert!(self.position == 0); - - if self.position + bytes.len() < self.buffer.len() { - &mut self.buffer[self.position..self.position + bytes.len()].copy_from_slice(bytes); - self.position += bytes.len(); - return Ok(()); - } - - match self.target { - OutputTarget::Bytes => { - unreachable!(); - } - OutputTarget::Write(ref mut write, _) => { - write.write_all(bytes)?; - } - OutputTarget::Vec(ref mut vec) => { - vec.extend(bytes); - unsafe { - self.buffer = remove_lifetime_mut(remaining_capacity_as_slice_mut(vec)); - } - } - } - Ok(()) - } - - /// Write a tag - pub fn write_tag( - &mut self, - field_number: u32, - wire_type: wire_format::WireType, - ) -> ProtobufResult<()> { - self.write_raw_varint32(wire_format::Tag::make(field_number, wire_type).value()) - } - - /// Write varint - pub fn write_raw_varint32(&mut self, value: u32) -> ProtobufResult<()> { - if self.buffer.len() - self.position >= 5 { - // fast path - let len = varint::encode_varint32(value, &mut self.buffer[self.position..]); - self.position += len; - Ok(()) - } else { - // slow path - let buf = &mut [0u8; 5]; - let len = varint::encode_varint32(value, buf); - self.write_raw_bytes(&buf[..len]) - } - } - - /// Write varint - pub fn write_raw_varint64(&mut self, value: u64) -> ProtobufResult<()> { - if self.buffer.len() - self.position >= 10 { - // fast path - let len = varint::encode_varint64(value, &mut self.buffer[self.position..]); - self.position += len; - Ok(()) - } else { - // slow path - let buf = &mut [0u8; 10]; - let len = varint::encode_varint64(value, buf); - self.write_raw_bytes(&buf[..len]) - } - } - - /// Write 32-bit integer little endian - pub fn write_raw_little_endian32(&mut self, value: u32) -> ProtobufResult<()> { - let bytes = unsafe { mem::transmute::<_, [u8; 4]>(value.to_le()) }; - self.write_raw_bytes(&bytes) - } - - /// Write 64-bit integer little endian - pub fn write_raw_little_endian64(&mut self, value: u64) -> ProtobufResult<()> { - let bytes = unsafe { mem::transmute::<_, [u8; 8]>(value.to_le()) }; - self.write_raw_bytes(&bytes) - } - - /// Write `float` - pub fn write_float_no_tag(&mut self, value: f32) -> ProtobufResult<()> { - let bits = unsafe { mem::transmute::(value) }; - self.write_raw_little_endian32(bits) - } - - /// Write `double` - pub fn write_double_no_tag(&mut self, value: f64) -> ProtobufResult<()> { - let bits = unsafe { mem::transmute::(value) }; - self.write_raw_little_endian64(bits) - } - - /// Write `float` field - pub fn write_float(&mut self, field_number: u32, value: f32) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeFixed32)?; - self.write_float_no_tag(value)?; - Ok(()) - } - - /// Write `double` field - pub fn write_double(&mut self, field_number: u32, value: f64) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeFixed64)?; - self.write_double_no_tag(value)?; - Ok(()) - } - - /// Write varint - pub fn write_uint64_no_tag(&mut self, value: u64) -> ProtobufResult<()> { - self.write_raw_varint64(value) - } - - /// Write varint - pub fn write_uint32_no_tag(&mut self, value: u32) -> ProtobufResult<()> { - self.write_raw_varint32(value) - } - - /// Write varint - pub fn write_int64_no_tag(&mut self, value: i64) -> ProtobufResult<()> { - self.write_raw_varint64(value as u64) - } - - /// Write varint - pub fn write_int32_no_tag(&mut self, value: i32) -> ProtobufResult<()> { - self.write_raw_varint64(value as u64) - } - - /// Write zigzag varint - pub fn write_sint64_no_tag(&mut self, value: i64) -> ProtobufResult<()> { - self.write_uint64_no_tag(encode_zig_zag_64(value)) - } - - /// Write zigzag varint - pub fn write_sint32_no_tag(&mut self, value: i32) -> ProtobufResult<()> { - self.write_uint32_no_tag(encode_zig_zag_32(value)) - } - - /// Write `fixed64` - pub fn write_fixed64_no_tag(&mut self, value: u64) -> ProtobufResult<()> { - self.write_raw_little_endian64(value) - } - - /// Write `fixed32` - pub fn write_fixed32_no_tag(&mut self, value: u32) -> ProtobufResult<()> { - self.write_raw_little_endian32(value) - } - - /// Write `sfixed64` - pub fn write_sfixed64_no_tag(&mut self, value: i64) -> ProtobufResult<()> { - self.write_raw_little_endian64(value as u64) - } - - /// Write `sfixed32` - pub fn write_sfixed32_no_tag(&mut self, value: i32) -> ProtobufResult<()> { - self.write_raw_little_endian32(value as u32) - } - - /// Write `bool` - pub fn write_bool_no_tag(&mut self, value: bool) -> ProtobufResult<()> { - self.write_raw_varint32(if value { 1 } else { 0 }) - } - - /// Write `enum` - pub fn write_enum_no_tag(&mut self, value: i32) -> ProtobufResult<()> { - self.write_int32_no_tag(value) - } - - /// Write `enum` - pub fn write_enum_obj_no_tag(&mut self, value: E) -> ProtobufResult<()> - where - E: ProtobufEnum, - { - self.write_enum_no_tag(value.value()) - } - - /// Write unknown value - pub fn write_unknown_no_tag(&mut self, unknown: UnknownValueRef) -> ProtobufResult<()> { - match unknown { - UnknownValueRef::Fixed64(fixed64) => self.write_raw_little_endian64(fixed64), - UnknownValueRef::Fixed32(fixed32) => self.write_raw_little_endian32(fixed32), - UnknownValueRef::Varint(varint) => self.write_raw_varint64(varint), - UnknownValueRef::LengthDelimited(bytes) => self.write_bytes_no_tag(bytes), - } - } - - /// Write `uint64` field - pub fn write_uint64(&mut self, field_number: u32, value: u64) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeVarint)?; - self.write_uint64_no_tag(value)?; - Ok(()) - } - - /// Write `uint32` field - pub fn write_uint32(&mut self, field_number: u32, value: u32) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeVarint)?; - self.write_uint32_no_tag(value)?; - Ok(()) - } - - /// Write `int64` field - pub fn write_int64(&mut self, field_number: u32, value: i64) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeVarint)?; - self.write_int64_no_tag(value)?; - Ok(()) - } - - /// Write `int32` field - pub fn write_int32(&mut self, field_number: u32, value: i32) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeVarint)?; - self.write_int32_no_tag(value)?; - Ok(()) - } - - /// Write `sint64` field - pub fn write_sint64(&mut self, field_number: u32, value: i64) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeVarint)?; - self.write_sint64_no_tag(value)?; - Ok(()) - } - - /// Write `sint32` field - pub fn write_sint32(&mut self, field_number: u32, value: i32) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeVarint)?; - self.write_sint32_no_tag(value)?; - Ok(()) - } - - /// Write `fixed64` field - pub fn write_fixed64(&mut self, field_number: u32, value: u64) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeFixed64)?; - self.write_fixed64_no_tag(value)?; - Ok(()) - } - - /// Write `fixed32` field - pub fn write_fixed32(&mut self, field_number: u32, value: u32) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeFixed32)?; - self.write_fixed32_no_tag(value)?; - Ok(()) - } - - /// Write `sfixed64` field - pub fn write_sfixed64(&mut self, field_number: u32, value: i64) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeFixed64)?; - self.write_sfixed64_no_tag(value)?; - Ok(()) - } - - /// Write `sfixed32` field - pub fn write_sfixed32(&mut self, field_number: u32, value: i32) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeFixed32)?; - self.write_sfixed32_no_tag(value)?; - Ok(()) - } - - /// Write `bool` field - pub fn write_bool(&mut self, field_number: u32, value: bool) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeVarint)?; - self.write_bool_no_tag(value)?; - Ok(()) - } - - /// Write `enum` field - pub fn write_enum(&mut self, field_number: u32, value: i32) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeVarint)?; - self.write_enum_no_tag(value)?; - Ok(()) - } - - /// Write `enum` field - pub fn write_enum_obj(&mut self, field_number: u32, value: E) -> ProtobufResult<()> - where - E: ProtobufEnum, - { - self.write_enum(field_number, value.value()) - } - - /// Write unknown field - pub fn write_unknown( - &mut self, - field_number: u32, - value: UnknownValueRef, - ) -> ProtobufResult<()> { - self.write_tag(field_number, value.wire_type())?; - self.write_unknown_no_tag(value)?; - Ok(()) - } - - /// Write unknown fields - pub fn write_unknown_fields(&mut self, fields: &UnknownFields) -> ProtobufResult<()> { - for (number, values) in fields { - for value in values { - self.write_unknown(number, value)?; - } - } - Ok(()) - } - - /// Write bytes - pub fn write_bytes_no_tag(&mut self, bytes: &[u8]) -> ProtobufResult<()> { - self.write_raw_varint32(bytes.len() as u32)?; - self.write_raw_bytes(bytes)?; - Ok(()) - } - - /// Write string - pub fn write_string_no_tag(&mut self, s: &str) -> ProtobufResult<()> { - self.write_bytes_no_tag(s.as_bytes()) - } - - /// Write message - pub fn write_message_no_tag(&mut self, msg: &M) -> ProtobufResult<()> { - msg.write_length_delimited_to(self) - } - - /// Write `bytes` field - pub fn write_bytes(&mut self, field_number: u32, bytes: &[u8]) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeLengthDelimited)?; - self.write_bytes_no_tag(bytes)?; - Ok(()) - } - - /// Write `string` field - pub fn write_string(&mut self, field_number: u32, s: &str) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeLengthDelimited)?; - self.write_string_no_tag(s)?; - Ok(()) - } - - /// Write `message` field - pub fn write_message(&mut self, field_number: u32, msg: &M) -> ProtobufResult<()> { - self.write_tag(field_number, wire_format::WireTypeLengthDelimited)?; - self.write_message_no_tag(msg)?; - Ok(()) - } -} - -impl<'a> Write for CodedOutputStream<'a> { - fn write(&mut self, buf: &[u8]) -> io::Result { - self.write_raw_bytes(buf)?; - Ok(buf.len()) - } - - fn flush(&mut self) -> io::Result<()> { - CodedOutputStream::flush(self).map_err(Into::into) - } -} - -#[cfg(test)] -mod test { - - use std::fmt::Debug; - use std::io; - use std::io::BufRead; - use std::io::Read; - use std::io::Write; - use std::iter::repeat; - - use crate::error::ProtobufError; - use crate::error::ProtobufResult; - use crate::hex::decode_hex; - use crate::hex::encode_hex; - - use super::wire_format; - use super::CodedInputStream; - use super::CodedOutputStream; - use super::READ_RAW_BYTES_MAX_ALLOC; - - fn test_read_partial(hex: &str, mut callback: F) - where - F: FnMut(&mut CodedInputStream), - { - let d = decode_hex(hex); - let mut reader = io::Cursor::new(d); - let mut is = CodedInputStream::from_buffered_reader(&mut reader as &mut dyn BufRead); - assert_eq!(0, is.pos()); - callback(&mut is); - } - - fn test_read(hex: &str, mut callback: F) - where - F: FnMut(&mut CodedInputStream), - { - let len = decode_hex(hex).len(); - test_read_partial(hex, |reader| { - callback(reader); - assert!(reader.eof().expect("eof")); - assert_eq!(len as u64, reader.pos()); - }); - } - - fn test_read_v(hex: &str, v: V, mut callback: F) - where - F: FnMut(&mut CodedInputStream) -> ProtobufResult, - V: PartialEq + Debug, - { - test_read(hex, |reader| { - assert_eq!(v, callback(reader).unwrap()); - }); - } - - #[test] - fn test_input_stream_read_raw_byte() { - test_read("17", |is| { - assert_eq!(23, is.read_raw_byte().unwrap()); - }); - } - - #[test] - fn test_input_stream_read_raw_varint() { - test_read_v("07", 7, |reader| reader.read_raw_varint32()); - test_read_v("07", 7, |reader| reader.read_raw_varint64()); - - test_read_v("96 01", 150, |reader| reader.read_raw_varint32()); - test_read_v("96 01", 150, |reader| reader.read_raw_varint64()); - - test_read_v( - "ff ff ff ff ff ff ff ff ff 01", - 0xffffffffffffffff, - |reader| reader.read_raw_varint64(), - ); - - test_read_v("ff ff ff ff 0f", 0xffffffff, |reader| { - reader.read_raw_varint32() - }); - test_read_v("ff ff ff ff 0f", 0xffffffff, |reader| { - reader.read_raw_varint64() - }); - } - - #[test] - fn test_input_stream_read_raw_vaint_malformed() { - // varint cannot have length > 10 - test_read_partial("ff ff ff ff ff ff ff ff ff ff 01", |reader| { - let result = reader.read_raw_varint64(); - match result { - // TODO: make an enum variant - Err(ProtobufError::WireError(..)) => (), - _ => panic!(), - } - }); - test_read_partial("ff ff ff ff ff ff ff ff ff ff 01", |reader| { - let result = reader.read_raw_varint32(); - match result { - // TODO: make an enum variant - Err(ProtobufError::WireError(..)) => (), - _ => panic!(), - } - }); - } - - #[test] - fn test_input_stream_read_raw_varint_unexpected_eof() { - test_read_partial("96 97", |reader| { - let result = reader.read_raw_varint32(); - match result { - Err(ProtobufError::WireError(..)) => (), - _ => panic!(), - } - }); - } - - #[test] - fn test_input_stream_read_raw_varint_pos() { - test_read_partial("95 01 98", |reader| { - assert_eq!(149, reader.read_raw_varint32().unwrap()); - assert_eq!(2, reader.pos()); - }); - } - - #[test] - fn test_input_stream_read_int32() { - test_read_v("02", 2, |reader| reader.read_int32()); - } - - #[test] - fn test_input_stream_read_float() { - test_read_v("95 73 13 61", 17e19, |is| is.read_float()); - } - - #[test] - fn test_input_stream_read_double() { - test_read_v("40 d5 ab 68 b3 07 3d 46", 23e29, |is| is.read_double()); - } - - #[test] - fn test_input_stream_skip_raw_bytes() { - test_read("", |reader| { - reader.skip_raw_bytes(0).unwrap(); - }); - test_read("aa bb", |reader| { - reader.skip_raw_bytes(2).unwrap(); - }); - test_read("aa bb cc dd ee ff", |reader| { - reader.skip_raw_bytes(6).unwrap(); - }); - } - - #[test] - fn test_input_stream_read_raw_bytes() { - test_read("", |reader| { - assert_eq!( - Vec::from(&b""[..]), - reader.read_raw_bytes(0).expect("read_raw_bytes") - ); - }) - } - - #[test] - fn test_input_stream_limits() { - test_read("aa bb cc", |is| { - let old_limit = is.push_limit(1).unwrap(); - assert_eq!(1, is.bytes_until_limit()); - let r1 = is.read_raw_bytes(1).unwrap(); - assert_eq!(&[0xaa as u8], &r1[..]); - is.pop_limit(old_limit); - let r2 = is.read_raw_bytes(2).unwrap(); - assert_eq!(&[0xbb as u8, 0xcc], &r2[..]); - }); - } - - #[test] - fn test_input_stream_io_read() { - test_read("aa bb cc", |is| { - let mut buf = [0; 3]; - assert_eq!(Read::read(is, &mut buf).expect("io::Read"), 3); - assert_eq!(buf, [0xaa, 0xbb, 0xcc]); - }); - } - - #[test] - fn test_input_stream_io_bufread() { - test_read("aa bb cc", |is| { - assert_eq!( - BufRead::fill_buf(is).expect("io::BufRead::fill_buf"), - &[0xaa, 0xbb, 0xcc] - ); - BufRead::consume(is, 3); - }); - } - - #[test] - fn test_input_stream_read_raw_bytes_into_huge() { - let mut v = Vec::new(); - for i in 0..READ_RAW_BYTES_MAX_ALLOC + 1000 { - v.push((i % 10) as u8); - } - - let mut slice: &[u8] = v.as_slice(); - - let mut is = CodedInputStream::new(&mut slice); - - let mut buf = Vec::new(); - - is.read_raw_bytes_into(READ_RAW_BYTES_MAX_ALLOC as u32 + 10, &mut buf) - .expect("read"); - - assert_eq!(READ_RAW_BYTES_MAX_ALLOC + 10, buf.len()); - - buf.clear(); - - is.read_raw_bytes_into(1000 - 10, &mut buf).expect("read"); - - assert_eq!(1000 - 10, buf.len()); - - assert!(is.eof().expect("eof")); - } - - fn test_write(expected: &str, mut gen: F) - where - F: FnMut(&mut CodedOutputStream) -> ProtobufResult<()>, - { - let expected_bytes = decode_hex(expected); - - // write to Write - { - let mut v = Vec::new(); - { - let mut os = CodedOutputStream::new(&mut v as &mut dyn Write); - gen(&mut os).unwrap(); - os.flush().unwrap(); - } - assert_eq!(encode_hex(&expected_bytes), encode_hex(&v)); - } - - // write to &[u8] - { - let mut r = Vec::with_capacity(expected_bytes.len()); - r.resize(expected_bytes.len(), 0); - { - let mut os = CodedOutputStream::bytes(&mut r); - gen(&mut os).unwrap(); - os.check_eof(); - } - assert_eq!(encode_hex(&expected_bytes), encode_hex(&r)); - } - - // write to Vec - { - let mut r = Vec::new(); - r.extend(&[11, 22, 33, 44, 55, 66, 77]); - { - let mut os = CodedOutputStream::vec(&mut r); - gen(&mut os).unwrap(); - os.flush().unwrap(); - } - - r.drain(..7); - assert_eq!(encode_hex(&expected_bytes), encode_hex(&r)); - } - } - - #[test] - fn test_output_stream_write_raw_byte() { - test_write("a1", |os| os.write_raw_byte(0xa1)); - } - - #[test] - fn test_output_stream_write_tag() { - test_write("08", |os| os.write_tag(1, wire_format::WireTypeVarint)); - } - - #[test] - fn test_output_stream_write_raw_bytes() { - test_write("00 ab", |os| os.write_raw_bytes(&[0x00, 0xab])); - - let expected = repeat("01 02 03 04") - .take(2048) - .collect::>() - .join(" "); - test_write(&expected, |os| { - for _ in 0..2048 { - os.write_raw_bytes(&[0x01, 0x02, 0x03, 0x04])?; - } - - Ok(()) - }); - } - - #[test] - fn test_output_stream_write_raw_varint32() { - test_write("96 01", |os| os.write_raw_varint32(150)); - test_write("ff ff ff ff 0f", |os| os.write_raw_varint32(0xffffffff)); - } - - #[test] - fn test_output_stream_write_raw_varint64() { - test_write("96 01", |os| os.write_raw_varint64(150)); - test_write("ff ff ff ff ff ff ff ff ff 01", |os| { - os.write_raw_varint64(0xffffffffffffffff) - }); - } - - #[test] - fn test_output_stream_write_int32_no_tag() { - test_write("ff ff ff ff ff ff ff ff ff 01", |os| { - os.write_int32_no_tag(-1) - }); - } - - #[test] - fn test_output_stream_write_int64_no_tag() { - test_write("ff ff ff ff ff ff ff ff ff 01", |os| { - os.write_int64_no_tag(-1) - }); - } - - #[test] - fn test_output_stream_write_raw_little_endian32() { - test_write("f1 e2 d3 c4", |os| os.write_raw_little_endian32(0xc4d3e2f1)); - } - - #[test] - fn test_output_stream_write_float_no_tag() { - test_write("95 73 13 61", |os| os.write_float_no_tag(17e19)); - } - - #[test] - fn test_output_stream_write_double_no_tag() { - test_write("40 d5 ab 68 b3 07 3d 46", |os| { - os.write_double_no_tag(23e29) - }); - } - - #[test] - fn test_output_stream_write_raw_little_endian64() { - test_write("f1 e2 d3 c4 b5 a6 07 f8", |os| { - os.write_raw_little_endian64(0xf807a6b5c4d3e2f1) - }); - } - - #[test] - fn test_output_stream_io_write() { - let expected = [0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77]; - - // write to Write - { - let mut v = Vec::new(); - { - let mut os = CodedOutputStream::new(&mut v as &mut dyn Write); - Write::write(&mut os, &expected).expect("io::Write::write"); - Write::flush(&mut os).expect("io::Write::flush"); - } - assert_eq!(expected, *v); - } - - // write to &[u8] - { - let mut v = Vec::with_capacity(expected.len()); - v.resize(expected.len(), 0); - { - let mut os = CodedOutputStream::bytes(&mut v); - Write::write(&mut os, &expected).expect("io::Write::write"); - Write::flush(&mut os).expect("io::Write::flush"); - os.check_eof(); - } - assert_eq!(expected, *v); - } - - // write to Vec - { - let mut v = Vec::new(); - { - let mut os = CodedOutputStream::vec(&mut v); - Write::write(&mut os, &expected).expect("io::Write::write"); - Write::flush(&mut os).expect("io::Write::flush"); - } - assert_eq!(expected, *v); - } - } -} diff --git a/src/types.rs b/src/types.rs index bcfdd8f..d9bbb10 100644 --- a/src/types.rs +++ b/src/types.rs @@ -8,13 +8,13 @@ use crate::chars::Chars; #[cfg(feature = "bytes")] use bytes::Bytes; +use crate::coded_input_stream::CodedInputStream; +use crate::coded_output_stream::CodedOutputStream; use crate::enums::ProtobufEnum; use crate::error::ProtobufResult; use crate::message::Message; use crate::reflect::ProtobufValue; use crate::rt; -use crate::stream::CodedInputStream; -use crate::stream::CodedOutputStream; use crate::unknown::UnknownValues; use crate::wire_format::WireType; use crate::zigzag::decode_zig_zag_32; diff --git a/src/unknown.rs b/src/unknown.rs index 241fb3c..fb0cf45 100644 --- a/src/unknown.rs +++ b/src/unknown.rs @@ -1,5 +1,5 @@ use crate::clear::Clear; -use crate::stream::wire_format; +use crate::wire_format; use crate::zigzag::encode_zig_zag_32; use crate::zigzag::encode_zig_zag_64; use std::collections::hash_map; diff --git a/src/well_known_types/any.rs b/src/well_known_types/any.rs index 397f9db..f306202 100644 --- a/src/well_known_types/any.rs +++ b/src/well_known_types/any.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/well_known_types/api.rs b/src/well_known_types/api.rs index cb71f3b..34798b4 100644 --- a/src/well_known_types/api.rs +++ b/src/well_known_types/api.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/well_known_types/duration.rs b/src/well_known_types/duration.rs index d810b14..ec82f78 100644 --- a/src/well_known_types/duration.rs +++ b/src/well_known_types/duration.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/well_known_types/empty.rs b/src/well_known_types/empty.rs index 2464f43..5c69487 100644 --- a/src/well_known_types/empty.rs +++ b/src/well_known_types/empty.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/well_known_types/field_mask.rs b/src/well_known_types/field_mask.rs index dc0ed10..9d1f8f6 100644 --- a/src/well_known_types/field_mask.rs +++ b/src/well_known_types/field_mask.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/well_known_types/source_context.rs b/src/well_known_types/source_context.rs index 2086b35..55c1ab4 100644 --- a/src/well_known_types/source_context.rs +++ b/src/well_known_types/source_context.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/well_known_types/struct_pb.rs b/src/well_known_types/struct_pb.rs index 613fef3..222d712 100644 --- a/src/well_known_types/struct_pb.rs +++ b/src/well_known_types/struct_pb.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/well_known_types/timestamp.rs b/src/well_known_types/timestamp.rs index 0d48c7e..d0f7903 100644 --- a/src/well_known_types/timestamp.rs +++ b/src/well_known_types/timestamp.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/well_known_types/type_pb.rs b/src/well_known_types/type_pb.rs index 88744ae..d6c06cc 100644 --- a/src/well_known_types/type_pb.rs +++ b/src/well_known_types/type_pb.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/well_known_types/wrappers.rs b/src/well_known_types/wrappers.rs index 76392a2..11edf4c 100644 --- a/src/well_known_types/wrappers.rs +++ b/src/well_known_types/wrappers.rs @@ -1,4 +1,4 @@ -// This file is generated by rust-protobuf 2.22.0-pre. Do not edit +// This file is generated by rust-protobuf 2.22.0. Do not edit // @generated // https://github.com/rust-lang/rust-clippy/issues/702 @@ -6,7 +6,7 @@ #![allow(clippy::all)] #![allow(unused_attributes)] -#![rustfmt::skip] +#![cfg_attr(rustfmt, rustfmt::skip)] #![allow(box_pointers)] #![allow(dead_code)] diff --git a/src/wire_format.rs b/src/wire_format.rs new file mode 100644 index 0000000..9f54af3 --- /dev/null +++ b/src/wire_format.rs @@ -0,0 +1,105 @@ +//! Serialization constants. + +// TODO: temporary +pub use self::WireType::*; + +/// Tag occupies 3 bits +pub const TAG_TYPE_BITS: u32 = 3; +/// Tag mask +pub const TAG_TYPE_MASK: u32 = (1u32 << TAG_TYPE_BITS) - 1; +/// Max possible field number +pub const FIELD_NUMBER_MAX: u32 = 0x1fffffff; + +/// One of six defined protobuf wire types +#[derive(PartialEq, Eq, Clone, Debug)] +pub enum WireType { + /// Varint (e. g. `int32` or `sint64`) + WireTypeVarint = 0, + /// Fixed size 64 bit (e. g. `fixed64` or `double`) + WireTypeFixed64 = 1, + /// Length-delimited (e. g. `message` or `string`) + WireTypeLengthDelimited = 2, + /// Groups are not supported by rust-protobuf + WireTypeStartGroup = 3, + /// Groups are not supported by rust-protobuf + WireTypeEndGroup = 4, + /// Fixed size 64 bit (e. g. `fixed32` or `float`) + WireTypeFixed32 = 5, +} + +impl Copy for WireType {} + +impl WireType { + /// Parse wire type + pub fn new(n: u32) -> Option { + match n { + 0 => Some(WireTypeVarint), + 1 => Some(WireTypeFixed64), + 2 => Some(WireTypeLengthDelimited), + 3 => Some(WireTypeStartGroup), + 4 => Some(WireTypeEndGroup), + 5 => Some(WireTypeFixed32), + _ => None, + } + } +} + +/// Parsed protobuf tag, which is a pair of field number and wire type +#[derive(Clone)] +pub struct Tag { + field_number: u32, + wire_type: WireType, +} + +impl Copy for Tag {} + +impl Tag { + /// Pack a tag to integer + pub fn value(self) -> u32 { + (self.field_number << TAG_TYPE_BITS) | (self.wire_type as u32) + } + + /// Parse integer into `Tag` object + // TODO: should return Result instead of Option + pub fn new(value: u32) -> Option { + let wire_type = WireType::new(value & TAG_TYPE_MASK); + if wire_type.is_none() { + return None; + } + let field_number = value >> TAG_TYPE_BITS; + if field_number == 0 { + return None; + } + Some(Tag { + field_number: field_number, + wire_type: wire_type.unwrap(), + }) + } + + /// Create a tag from a field number and wire type. + /// + /// # Panics + /// + /// If field number is outside of allowed range. + pub fn make(field_number: u32, wire_type: WireType) -> Tag { + assert!(field_number > 0 && field_number <= FIELD_NUMBER_MAX); + Tag { + field_number: field_number, + wire_type: wire_type, + } + } + + /// Tag as pair of (field number, wire type) + pub fn unpack(self) -> (u32, WireType) { + (self.field_number(), self.wire_type()) + } + + fn wire_type(self) -> WireType { + self.wire_type + } + + /// Protobuf field number + pub fn field_number(self) -> u32 { + self.field_number + } +} -- cgit v1.2.3