diff options
145 files changed, 1238 insertions, 873 deletions
@@ -64,11 +64,11 @@ config_setting( ) # This should be updated along with build.yaml -g_stands_for = "gorgeous" +g_stands_for = "glorious" core_version = "6.0.0-dev" -version = "1.11.0-dev" +version = "1.12.0-dev" GPR_PUBLIC_HDRS = [ "include/grpc/support/alloc.h", @@ -141,7 +141,6 @@ GRPCXX_SRCS = [ "src/cpp/server/server_posix.cc", "src/cpp/thread_manager/thread_manager.cc", "src/cpp/util/byte_buffer_cc.cc", - "src/cpp/util/slice_cc.cc", "src/cpp/util/status.cc", "src/cpp/util/string_ref.cc", "src/cpp/util/time_cc.cc", @@ -245,6 +244,8 @@ GRPCXX_PUBLIC_HDRS = [ "include/grpcpp/support/byte_buffer.h", "include/grpcpp/support/channel_arguments.h", "include/grpcpp/support/config.h", + "include/grpcpp/support/proto_buffer_reader.h", + "include/grpcpp/support/proto_buffer_writer.h", "include/grpcpp/support/slice.h", "include/grpcpp/support/status.h", "include/grpcpp/support/status_code_enum.h", @@ -1868,6 +1869,8 @@ grpc_cc_library( language = "c++", public_hdrs = [ "include/grpc++/impl/codegen/proto_utils.h", + "include/grpcpp/impl/codegen/proto_buffer_reader.h", + "include/grpcpp/impl/codegen/proto_buffer_writer.h", "include/grpcpp/impl/codegen/proto_utils.h", ], deps = [ diff --git a/CMakeLists.txt b/CMakeLists.txt index 30c2023a2d..8b13afaa71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,7 +24,7 @@ cmake_minimum_required(VERSION 2.8) set(PACKAGE_NAME "grpc") -set(PACKAGE_VERSION "1.11.0-dev") +set(PACKAGE_VERSION "1.12.0-dev") set(PACKAGE_STRING "${PACKAGE_NAME} ${PACKAGE_VERSION}") set(PACKAGE_TARNAME "${PACKAGE_NAME}-${PACKAGE_VERSION}") set(PACKAGE_BUGREPORT "https://github.com/grpc/grpc/issues/") @@ -2714,7 +2714,6 @@ add_library(grpc++ src/cpp/server/server_posix.cc src/cpp/thread_manager/thread_manager.cc src/cpp/util/byte_buffer_cc.cc - src/cpp/util/slice_cc.cc src/cpp/util/status.cc src/cpp/util/string_ref.cc src/cpp/util/time_cc.cc @@ -2839,6 +2838,8 @@ foreach(_hdr include/grpcpp/support/byte_buffer.h include/grpcpp/support/channel_arguments.h include/grpcpp/support/config.h + include/grpcpp/support/proto_buffer_reader.h + include/grpcpp/support/proto_buffer_writer.h include/grpcpp/support/slice.h include/grpcpp/support/status.h include/grpcpp/support/status_code_enum.h @@ -2957,6 +2958,8 @@ foreach(_hdr include/grpcpp/impl/codegen/sync_stream.h include/grpcpp/impl/codegen/time.h include/grpc++/impl/codegen/proto_utils.h + include/grpcpp/impl/codegen/proto_buffer_reader.h + include/grpcpp/impl/codegen/proto_buffer_writer.h include/grpcpp/impl/codegen/proto_utils.h include/grpc++/impl/codegen/config_protobuf.h include/grpcpp/impl/codegen/config_protobuf.h @@ -3065,7 +3068,6 @@ add_library(grpc++_cronet src/cpp/server/server_posix.cc src/cpp/thread_manager/thread_manager.cc src/cpp/util/byte_buffer_cc.cc - src/cpp/util/slice_cc.cc src/cpp/util/status.cc src/cpp/util/string_ref.cc src/cpp/util/time_cc.cc @@ -3400,6 +3402,8 @@ foreach(_hdr include/grpcpp/support/byte_buffer.h include/grpcpp/support/channel_arguments.h include/grpcpp/support/config.h + include/grpcpp/support/proto_buffer_reader.h + include/grpcpp/support/proto_buffer_writer.h include/grpcpp/support/slice.h include/grpcpp/support/status.h include/grpcpp/support/status_code_enum.h @@ -3943,6 +3947,8 @@ foreach(_hdr include/grpc/impl/codegen/sync_posix.h include/grpc/impl/codegen/sync_windows.h include/grpc++/impl/codegen/proto_utils.h + include/grpcpp/impl/codegen/proto_buffer_reader.h + include/grpcpp/impl/codegen/proto_buffer_writer.h include/grpcpp/impl/codegen/proto_utils.h include/grpc++/impl/codegen/config_protobuf.h include/grpcpp/impl/codegen/config_protobuf.h @@ -4116,6 +4122,8 @@ foreach(_hdr include/grpc/impl/codegen/sync_posix.h include/grpc/impl/codegen/sync_windows.h include/grpc++/impl/codegen/proto_utils.h + include/grpcpp/impl/codegen/proto_buffer_reader.h + include/grpcpp/impl/codegen/proto_buffer_writer.h include/grpcpp/impl/codegen/proto_utils.h include/grpc++/impl/codegen/config_protobuf.h include/grpcpp/impl/codegen/config_protobuf.h @@ -4164,7 +4172,6 @@ add_library(grpc++_unsecure src/cpp/server/server_posix.cc src/cpp/thread_manager/thread_manager.cc src/cpp/util/byte_buffer_cc.cc - src/cpp/util/slice_cc.cc src/cpp/util/status.cc src/cpp/util/string_ref.cc src/cpp/util/time_cc.cc @@ -4288,6 +4295,8 @@ foreach(_hdr include/grpcpp/support/byte_buffer.h include/grpcpp/support/channel_arguments.h include/grpcpp/support/config.h + include/grpcpp/support/proto_buffer_reader.h + include/grpcpp/support/proto_buffer_writer.h include/grpcpp/support/slice.h include/grpcpp/support/status.h include/grpcpp/support/status_code_enum.h @@ -421,8 +421,8 @@ Q = @ endif CORE_VERSION = 6.0.0-dev -CPP_VERSION = 1.11.0-dev -CSHARP_VERSION = 1.11.0-dev +CPP_VERSION = 1.12.0-dev +CSHARP_VERSION = 1.12.0-dev CPPFLAGS_NO_ARCH += $(addprefix -I, $(INCLUDES)) $(addprefix -D, $(DEFINES)) CPPFLAGS += $(CPPFLAGS_NO_ARCH) $(ARCH_FLAGS) @@ -5017,7 +5017,6 @@ LIBGRPC++_SRC = \ src/cpp/server/server_posix.cc \ src/cpp/thread_manager/thread_manager.cc \ src/cpp/util/byte_buffer_cc.cc \ - src/cpp/util/slice_cc.cc \ src/cpp/util/status.cc \ src/cpp/util/string_ref.cc \ src/cpp/util/time_cc.cc \ @@ -5107,6 +5106,8 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/support/byte_buffer.h \ include/grpcpp/support/channel_arguments.h \ include/grpcpp/support/config.h \ + include/grpcpp/support/proto_buffer_reader.h \ + include/grpcpp/support/proto_buffer_writer.h \ include/grpcpp/support/slice.h \ include/grpcpp/support/status.h \ include/grpcpp/support/status_code_enum.h \ @@ -5225,6 +5226,8 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/impl/codegen/sync_stream.h \ include/grpcpp/impl/codegen/time.h \ include/grpc++/impl/codegen/proto_utils.h \ + include/grpcpp/impl/codegen/proto_buffer_reader.h \ + include/grpcpp/impl/codegen/proto_buffer_writer.h \ include/grpcpp/impl/codegen/proto_utils.h \ include/grpc++/impl/codegen/config_protobuf.h \ include/grpcpp/impl/codegen/config_protobuf.h \ @@ -5378,7 +5381,6 @@ LIBGRPC++_CRONET_SRC = \ src/cpp/server/server_posix.cc \ src/cpp/thread_manager/thread_manager.cc \ src/cpp/util/byte_buffer_cc.cc \ - src/cpp/util/slice_cc.cc \ src/cpp/util/status.cc \ src/cpp/util/string_ref.cc \ src/cpp/util/time_cc.cc \ @@ -5677,6 +5679,8 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/support/byte_buffer.h \ include/grpcpp/support/channel_arguments.h \ include/grpcpp/support/config.h \ + include/grpcpp/support/proto_buffer_reader.h \ + include/grpcpp/support/proto_buffer_writer.h \ include/grpcpp/support/slice.h \ include/grpcpp/support/status.h \ include/grpcpp/support/status_code_enum.h \ @@ -6205,6 +6209,8 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ include/grpc++/impl/codegen/proto_utils.h \ + include/grpcpp/impl/codegen/proto_buffer_reader.h \ + include/grpcpp/impl/codegen/proto_buffer_writer.h \ include/grpcpp/impl/codegen/proto_utils.h \ include/grpc++/impl/codegen/config_protobuf.h \ include/grpcpp/impl/codegen/config_protobuf.h \ @@ -6355,6 +6361,8 @@ PUBLIC_HEADERS_CXX += \ include/grpc/impl/codegen/sync_posix.h \ include/grpc/impl/codegen/sync_windows.h \ include/grpc++/impl/codegen/proto_utils.h \ + include/grpcpp/impl/codegen/proto_buffer_reader.h \ + include/grpcpp/impl/codegen/proto_buffer_writer.h \ include/grpcpp/impl/codegen/proto_utils.h \ include/grpc++/impl/codegen/config_protobuf.h \ include/grpcpp/impl/codegen/config_protobuf.h \ @@ -6442,7 +6450,6 @@ LIBGRPC++_UNSECURE_SRC = \ src/cpp/server/server_posix.cc \ src/cpp/thread_manager/thread_manager.cc \ src/cpp/util/byte_buffer_cc.cc \ - src/cpp/util/slice_cc.cc \ src/cpp/util/status.cc \ src/cpp/util/string_ref.cc \ src/cpp/util/time_cc.cc \ @@ -6532,6 +6539,8 @@ PUBLIC_HEADERS_CXX += \ include/grpcpp/support/byte_buffer.h \ include/grpcpp/support/channel_arguments.h \ include/grpcpp/support/config.h \ + include/grpcpp/support/proto_buffer_reader.h \ + include/grpcpp/support/proto_buffer_writer.h \ include/grpcpp/support/slice.h \ include/grpcpp/support/status.h \ include/grpcpp/support/status_code_enum.h \ diff --git a/PYTHON-MANIFEST.in b/PYTHON-MANIFEST.in index 8be0af964f..b30a1fbf90 100644 --- a/PYTHON-MANIFEST.in +++ b/PYTHON-MANIFEST.in @@ -4,10 +4,11 @@ graft src/python/grpcio/grpcio.egg-info graft src/core graft src/boringssl graft include/grpc +graft third_party/address_sorting graft third_party/boringssl +graft third_party/cares graft third_party/nanopb graft third_party/zlib -graft third_party/cares include src/python/grpcio/_spawn_patch.py include src/python/grpcio/commands.py include src/python/grpcio/grpc_version.py diff --git a/build.yaml b/build.yaml index b79b04f062..b6aa2af785 100644 --- a/build.yaml +++ b/build.yaml @@ -13,8 +13,8 @@ settings: '#09': Per-language overrides are possible with (eg) ruby_version tag here '#10': See the expand_version.py for all the quirks here core_version: 6.0.0-dev - g_stands_for: gorgeous - version: 1.11.0-dev + g_stands_for: glorious + version: 1.12.0-dev filegroups: - name: alts_proto headers: @@ -1162,6 +1162,8 @@ filegroups: language: c++ public_headers: - include/grpc++/impl/codegen/proto_utils.h + - include/grpcpp/impl/codegen/proto_buffer_reader.h + - include/grpcpp/impl/codegen/proto_buffer_writer.h - include/grpcpp/impl/codegen/proto_utils.h uses: - grpc++_codegen_base @@ -1252,6 +1254,8 @@ filegroups: - include/grpcpp/support/byte_buffer.h - include/grpcpp/support/channel_arguments.h - include/grpcpp/support/config.h + - include/grpcpp/support/proto_buffer_reader.h + - include/grpcpp/support/proto_buffer_writer.h - include/grpcpp/support/slice.h - include/grpcpp/support/status.h - include/grpcpp/support/status_code_enum.h @@ -1298,7 +1302,6 @@ filegroups: - src/cpp/server/server_posix.cc - src/cpp/thread_manager/thread_manager.cc - src/cpp/util/byte_buffer_cc.cc - - src/cpp/util/slice_cc.cc - src/cpp/util/status.cc - src/cpp/util/string_ref.cc - src/cpp/util/time_cc.cc diff --git a/config.w32 b/config.w32 index 2c6516fe3a..f60a5b746d 100644 --- a/config.w32 +++ b/config.w32 @@ -636,11 +636,13 @@ if (PHP_GRPC != "no") { EXTENSION("grpc", grpc_source, null, "/DOPENSSL_NO_ASM /D_GNU_SOURCE /DWIN32_LEAN_AND_MEAN "+ "/D_HAS_EXCEPTIONS=0 /DNOMINMAX /DGRPC_ARES=0 /D_WIN32_WINNT=0x600 "+ + "/DPB_FIELD_16BIT "+ "/I"+configure_module_dirname+" "+ "/I"+configure_module_dirname+"\\include "+ "/I"+configure_module_dirname+"\\src\\php\\ext\\grpc "+ "/I"+configure_module_dirname+"\\third_party\\boringssl\\include "+ - "/I"+configure_module_dirname+"\\third_party\\zlib"); + "/I"+configure_module_dirname+"\\third_party\\zlib "+ + "/I"+configure_module_dirname+"\\third_party\\address_sorting\\include"); base_dir = get_define('BUILD_DIR'); FSO.CreateFolder(base_dir+"\\ext"); diff --git a/doc/g_stands_for.md b/doc/g_stands_for.md index 47ae7c5462..ee8a827644 100644 --- a/doc/g_stands_for.md +++ b/doc/g_stands_for.md @@ -15,3 +15,4 @@ future), and the corresponding version numbers that used them: - 1.9 'g' stands for 'glossy' - 1.10 'g' stands for 'glamorous' - 1.11 'g' stands for 'gorgeous' +- 1.12 'g' stands for 'glorious' diff --git a/examples/android/helloworld/app/CMakeLists.txt b/examples/android/helloworld/app/CMakeLists.txt index ad6c1b834c..f449a5ce2e 100644 --- a/examples/android/helloworld/app/CMakeLists.txt +++ b/examples/android/helloworld/app/CMakeLists.txt @@ -12,40 +12,6 @@ file(MAKE_DIRECTORY ${GRPC_BUILD_DIR}) add_subdirectory(${GRPC_SRC_DIR} ${GRPC_BUILD_DIR}) -include_directories(${GRPC_SRC_DIR}/include) - -add_library(libgrpc STATIC IMPORTED) -set_target_properties(libgrpc PROPERTIES IMPORTED_LOCATION - ${GRPC_BUILD_DIR}/libgrpc.a) - -add_library(libgrpc++ STATIC IMPORTED) -set_target_properties(libgrpc++ PROPERTIES IMPORTED_LOCATION - ${GRPC_BUILD_DIR}/libgrpc++.a) - -add_library(libgpr STATIC IMPORTED) -set_target_properties(libgpr PROPERTIES IMPORTED_LOCATION - ${GRPC_BUILD_DIR}/libgpr.a) - -add_library(libaddress_sorting STATIC IMPORTED) -set_target_properties(libaddress_sorting PROPERTIES IMPORTED_LOCATION - ${GRPC_BUILD_DIR}/libaddress_sorting.a) - -add_library(libcares STATIC IMPORTED) -set_target_properties(libcares PROPERTIES IMPORTED_LOCATION - ${GRPC_BUILD_DIR}/third_party/cares/cares/lib/libcares.a) - -add_library(libzlib STATIC IMPORTED) -set_target_properties(libzlib PROPERTIES IMPORTED_LOCATION - ${GRPC_BUILD_DIR}/third_party/zlib/libz.a) - -add_library(libcrypto STATIC IMPORTED) -set_target_properties(libcrypto PROPERTIES IMPORTED_LOCATION - ${GRPC_BUILD_DIR}/third_party/boringssl/crypto/libcrypto.a) - -add_library(libssl STATIC IMPORTED) -set_target_properties(libssl PROPERTIES IMPORTED_LOCATION - ${GRPC_BUILD_DIR}/third_party/boringssl/ssl/libssl.a) - set(GRPC_PROTO_GENS_DIR ${CMAKE_BINARY_DIR}/gens) file(MAKE_DIRECTORY ${GRPC_PROTO_GENS_DIR}) include_directories(${GRPC_PROTO_GENS_DIR}) @@ -100,8 +66,8 @@ add_library(helloworld_proto_lib SHARED ${HELLOWORLD_PROTO_HDRS} ${HELLOWORLD_PROTO_SRCS}) target_link_libraries(helloworld_proto_lib + grpc++ libprotobuf - libgrpc++ android log) @@ -115,14 +81,6 @@ target_include_directories(grpc-helloworld PRIVATE ${HELLOWORLD_PROTO_HEADERS}) target_link_libraries(grpc-helloworld - libgrpc++ - libgrpc - libaddress_sorting - libzlib - libcares - libssl - libcrypto helloworld_proto_lib - libgpr android ${log-lib}) diff --git a/gRPC-C++.podspec b/gRPC-C++.podspec index 9d4d15a9a1..990b0a4f9d 100644 --- a/gRPC-C++.podspec +++ b/gRPC-C++.podspec @@ -23,7 +23,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-C++' # TODO (mxyan): use version that match gRPC version when pod is stabilized - # version = '1.11.0-dev' + # version = '1.12.0-dev' version = '0.0.2' s.version = version s.summary = 'gRPC C++ library' @@ -31,7 +31,7 @@ Pod::Spec.new do |s| s.license = 'Apache License, Version 2.0' s.authors = { 'The gRPC contributors' => 'grpc-packages@google.com' } - grpc_version = '1.11.0-dev' + grpc_version = '1.12.0-dev' s.source = { :git => 'https://github.com/grpc/grpc.git', @@ -112,6 +112,8 @@ Pod::Spec.new do |s| 'include/grpcpp/support/byte_buffer.h', 'include/grpcpp/support/channel_arguments.h', 'include/grpcpp/support/config.h', + 'include/grpcpp/support/proto_buffer_reader.h', + 'include/grpcpp/support/proto_buffer_writer.h', 'include/grpcpp/support/slice.h', 'include/grpcpp/support/status.h', 'include/grpcpp/support/status_code_enum.h', @@ -206,7 +208,6 @@ Pod::Spec.new do |s| 'src/cpp/server/server_posix.cc', 'src/cpp/thread_manager/thread_manager.cc', 'src/cpp/util/byte_buffer_cc.cc', - 'src/cpp/util/slice_cc.cc', 'src/cpp/util/status.cc', 'src/cpp/util/string_ref.cc', 'src/cpp/util/time_cc.cc', diff --git a/gRPC-Core.podspec b/gRPC-Core.podspec index d60a7f12f3..2899965512 100644 --- a/gRPC-Core.podspec +++ b/gRPC-Core.podspec @@ -22,7 +22,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-Core' - version = '1.11.0-dev' + version = '1.12.0-dev' s.version = version s.summary = 'Core cross-platform gRPC library, written in C' s.homepage = 'https://grpc.io' diff --git a/gRPC-ProtoRPC.podspec b/gRPC-ProtoRPC.podspec index 149687e5b4..e98dee4a3c 100644 --- a/gRPC-ProtoRPC.podspec +++ b/gRPC-ProtoRPC.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-ProtoRPC' - version = '1.11.0-dev' + version = '1.12.0-dev' s.version = version s.summary = 'RPC library for Protocol Buffers, based on gRPC' s.homepage = 'https://grpc.io' diff --git a/gRPC-RxLibrary.podspec b/gRPC-RxLibrary.podspec index 2497174417..3bba14b695 100644 --- a/gRPC-RxLibrary.podspec +++ b/gRPC-RxLibrary.podspec @@ -21,7 +21,7 @@ Pod::Spec.new do |s| s.name = 'gRPC-RxLibrary' - version = '1.11.0-dev' + version = '1.12.0-dev' s.version = version s.summary = 'Reactive Extensions library for iOS/OSX.' s.homepage = 'https://grpc.io' diff --git a/gRPC.podspec b/gRPC.podspec index 68e06b5536..bf73ffd22e 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -20,7 +20,7 @@ Pod::Spec.new do |s| s.name = 'gRPC' - version = '1.11.0-dev' + version = '1.12.0-dev' s.version = version s.summary = 'gRPC client library for iOS/OSX' s.homepage = 'https://grpc.io' @@ -1340,7 +1340,6 @@ 'src/cpp/server/server_posix.cc', 'src/cpp/thread_manager/thread_manager.cc', 'src/cpp/util/byte_buffer_cc.cc', - 'src/cpp/util/slice_cc.cc', 'src/cpp/util/status.cc', 'src/cpp/util/string_ref.cc', 'src/cpp/util/time_cc.cc', @@ -1488,7 +1487,6 @@ 'src/cpp/server/server_posix.cc', 'src/cpp/thread_manager/thread_manager.cc', 'src/cpp/util/byte_buffer_cc.cc', - 'src/cpp/util/slice_cc.cc', 'src/cpp/util/status.cc', 'src/cpp/util/string_ref.cc', 'src/cpp/util/time_cc.cc', diff --git a/include/grpcpp/client_context.h b/include/grpcpp/client_context.h index 8c12577a17..1994fcc6bc 100644 --- a/include/grpcpp/client_context.h +++ b/include/grpcpp/client_context.h @@ -26,7 +26,7 @@ /// /// Context settings are only relevant to the call they are invoked with, that /// is to say, they aren't sticky. Some of these settings, such as the -/// compression options, can be made persistant at channel construction time +/// compression options, can be made persistent at channel construction time /// (see \a grpc::CreateCustomChannel). /// /// \warning ClientContext instances should \em not be reused across rpcs. diff --git a/include/grpcpp/impl/codegen/byte_buffer.h b/include/grpcpp/impl/codegen/byte_buffer.h index e637efeb85..fda13a04e6 100644 --- a/include/grpcpp/impl/codegen/byte_buffer.h +++ b/include/grpcpp/impl/codegen/byte_buffer.h @@ -32,6 +32,8 @@ namespace grpc { class ServerInterface; +class ByteBuffer; +class ServerInterface; namespace internal { class CallOpSendMessage; @@ -45,6 +47,7 @@ template <class ServiceType, class RequestType, class ResponseType> class ServerStreamingHandler; template <class R> class DeserializeFuncType; +class GrpcByteBufferPeer; } // namespace internal /// A sequence of bytes. class ByteBuffer final { @@ -53,7 +56,30 @@ class ByteBuffer final { ByteBuffer() : buffer_(nullptr) {} /// Construct buffer from \a slices, of which there are \a nslices. - ByteBuffer(const Slice* slices, size_t nslices); + ByteBuffer(const Slice* slices, size_t nslices) { + // The following assertions check that the representation of a grpc::Slice + // is identical to that of a grpc_slice: it has a grpc_slice field, and + // nothing else. + static_assert(std::is_same<decltype(slices[0].slice_), grpc_slice>::value, + "Slice must have same representation as grpc_slice"); + static_assert(sizeof(Slice) == sizeof(grpc_slice), + "Slice must have same representation as grpc_slice"); + // The following assertions check that the representation of a ByteBuffer is + // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, + // and nothing else. + static_assert(std::is_same<decltype(buffer_), grpc_byte_buffer*>::value, + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), + "ByteBuffer must have same representation as " + "grpc_byte_buffer*"); + // The const_cast is legal if grpc_raw_byte_buffer_create() does no more + // than its advertised side effect of increasing the reference count of the + // slices it processes, and such an increase does not affect the semantics + // seen by the caller of this constructor. + buffer_ = g_core_codegen_interface->grpc_raw_byte_buffer_create( + reinterpret_cast<grpc_slice*>(const_cast<Slice*>(slices)), nslices); + } /// Constuct a byte buffer by referencing elements of existing buffer /// \a buf. Wrapper of core function grpc_byte_buffer_copy @@ -90,10 +116,18 @@ class ByteBuffer final { void Release() { buffer_ = nullptr; } /// Buffer size in bytes. - size_t Length() const; + size_t Length() const { + return buffer_ == nullptr + ? 0 + : g_core_codegen_interface->grpc_byte_buffer_length(buffer_); + } /// Swap the state of *this and *other. - void Swap(ByteBuffer* other); + void Swap(ByteBuffer* other) { + grpc_byte_buffer* tmp = other->buffer_; + other->buffer_ = buffer_; + buffer_ = tmp; + } /// Is this ByteBuffer valid? bool Valid() const { return (buffer_ != nullptr); } @@ -112,6 +146,9 @@ class ByteBuffer final { friend class internal::ServerStreamingHandler; template <class R> friend class internal::DeserializeFuncType; + friend class GrpcProtoBufferReader; + friend class GrpcProtoBufferWriter; + friend class internal::GrpcByteBufferPeer; grpc_byte_buffer* buffer_; diff --git a/include/grpcpp/impl/codegen/core_codegen.h b/include/grpcpp/impl/codegen/core_codegen.h index 0ca4ad524c..e9df96bf04 100644 --- a/include/grpcpp/impl/codegen/core_codegen.h +++ b/include/grpcpp/impl/codegen/core_codegen.h @@ -73,6 +73,7 @@ class CoreCodegen final : public CoreCodegenInterface { grpc_byte_buffer* grpc_byte_buffer_copy(grpc_byte_buffer* bb) override; void grpc_byte_buffer_destroy(grpc_byte_buffer* bb) override; + size_t grpc_byte_buffer_length(grpc_byte_buffer* bb) override; int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader* reader, grpc_byte_buffer* buffer) override; @@ -86,6 +87,8 @@ class CoreCodegen final : public CoreCodegenInterface { grpc_slice grpc_slice_new_with_user_data(void* p, size_t len, void (*destroy)(void*), void* user_data) override; + grpc_slice grpc_slice_new_with_len(void* p, size_t len, + void (*destroy)(void*, size_t)) override; grpc_slice grpc_empty_slice() override; grpc_slice grpc_slice_malloc(size_t length) override; void grpc_slice_unref(grpc_slice slice) override; diff --git a/include/grpcpp/impl/codegen/core_codegen_interface.h b/include/grpcpp/impl/codegen/core_codegen_interface.h index d72f579c8e..1167a188a2 100644 --- a/include/grpcpp/impl/codegen/core_codegen_interface.h +++ b/include/grpcpp/impl/codegen/core_codegen_interface.h @@ -81,6 +81,8 @@ class CoreCodegenInterface { virtual grpc_byte_buffer* grpc_byte_buffer_copy(grpc_byte_buffer* bb) = 0; virtual void grpc_byte_buffer_destroy(grpc_byte_buffer* bb) = 0; + virtual size_t grpc_byte_buffer_length(grpc_byte_buffer* bb) + GRPC_MUST_USE_RESULT = 0; virtual int grpc_byte_buffer_reader_init(grpc_byte_buffer_reader* reader, grpc_byte_buffer* buffer) @@ -95,6 +97,9 @@ class CoreCodegenInterface { virtual grpc_slice grpc_slice_new_with_user_data(void* p, size_t len, void (*destroy)(void*), void* user_data) = 0; + virtual grpc_slice grpc_slice_new_with_len(void* p, size_t len, + void (*destroy)(void*, + size_t)) = 0; virtual grpc_call_error grpc_call_cancel_with_status(grpc_call* call, grpc_status_code status, const char* description, diff --git a/include/grpcpp/impl/codegen/proto_buffer_reader.h b/include/grpcpp/impl/codegen/proto_buffer_reader.h new file mode 100644 index 0000000000..149e05c74f --- /dev/null +++ b/include/grpcpp/impl/codegen/proto_buffer_reader.h @@ -0,0 +1,151 @@ +/* + * + * Copyright 2015 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCPP_IMPL_CODEGEN_PROTO_BUFFER_READER_H +#define GRPCPP_IMPL_CODEGEN_PROTO_BUFFER_READER_H + +#include <type_traits> + +#include <grpc/impl/codegen/byte_buffer_reader.h> +#include <grpc/impl/codegen/grpc_types.h> +#include <grpc/impl/codegen/slice.h> +#include <grpcpp/impl/codegen/byte_buffer.h> +#include <grpcpp/impl/codegen/config_protobuf.h> +#include <grpcpp/impl/codegen/core_codegen_interface.h> +#include <grpcpp/impl/codegen/serialization_traits.h> +#include <grpcpp/impl/codegen/status.h> + +/// This header provides an object that reads bytes directly from a +/// grpc::ByteBuffer, via the ZeroCopyInputStream interface + +namespace grpc { + +extern CoreCodegenInterface* g_core_codegen_interface; + +/// This is a specialization of the protobuf class ZeroCopyInputStream +/// The principle is to get one chunk of data at a time from the proto layer, +/// with options to backup (re-see some bytes) or skip (forward past some bytes) +/// +/// Read more about ZeroCopyInputStream interface here: +/// https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.io.zero_copy_stream#ZeroCopyInputStream +class GrpcProtoBufferReader : public ::grpc::protobuf::io::ZeroCopyInputStream { + public: + /// Constructs buffer reader from \a buffer. Will set \a status() to non ok + /// if \a buffer is invalid (the internal buffer has not been initialized). + explicit GrpcProtoBufferReader(ByteBuffer* buffer) + : byte_count_(0), backup_count_(0), status_() { + /// Implemented through a grpc_byte_buffer_reader which iterates + /// over the slices that make up a byte buffer + if (!buffer->Valid() || + !g_core_codegen_interface->grpc_byte_buffer_reader_init( + &reader_, buffer->c_buffer())) { + status_ = Status(StatusCode::INTERNAL, + "Couldn't initialize byte buffer reader"); + } + } + + ~GrpcProtoBufferReader() { + if (status_.ok()) { + g_core_codegen_interface->grpc_byte_buffer_reader_destroy(&reader_); + } + } + + /// Give the proto library a chunk of data from the stream. The caller + /// may safely read from data[0, size - 1]. + bool Next(const void** data, int* size) override { + if (!status_.ok()) { + return false; + } + /// If we have backed up previously, we need to return the backed-up slice + if (backup_count_ > 0) { + *data = GRPC_SLICE_START_PTR(slice_) + GRPC_SLICE_LENGTH(slice_) - + backup_count_; + GPR_CODEGEN_ASSERT(backup_count_ <= INT_MAX); + *size = (int)backup_count_; + backup_count_ = 0; + return true; + } + /// Otherwise get the next slice from the byte buffer reader + if (!g_core_codegen_interface->grpc_byte_buffer_reader_next(&reader_, + &slice_)) { + return false; + } + g_core_codegen_interface->grpc_slice_unref(slice_); + *data = GRPC_SLICE_START_PTR(slice_); + // On win x64, int is only 32bit + GPR_CODEGEN_ASSERT(GRPC_SLICE_LENGTH(slice_) <= INT_MAX); + byte_count_ += * size = (int)GRPC_SLICE_LENGTH(slice_); + return true; + } + + /// Returns the status of the buffer reader. + Status status() const { return status_; } + + /// The proto library calls this to indicate that we should back up \a count + /// bytes that have already been returned by the last call of Next. + /// So do the backup and have that ready for a later Next. + void BackUp(int count) override { + GPR_CODEGEN_ASSERT(count <= static_cast<int>(GRPC_SLICE_LENGTH(slice_))); + backup_count_ = count; + } + + /// The proto library calls this to skip over \a count bytes. Implement this + /// using Next and BackUp combined. + bool Skip(int count) override { + const void* data; + int size; + while (Next(&data, &size)) { + if (size >= count) { + BackUp(size - count); + return true; + } + // size < count; + count -= size; + } + // error or we have too large count; + return false; + } + + /// Returns the total number of bytes read since this object was created. + grpc::protobuf::int64 ByteCount() const override { + return byte_count_ - backup_count_; + } + + // These protected members are needed to support internal optimizations. + // they expose internal bits of grpc core that are NOT stable. If you have + // a use case needs to use one of these functions, please send an email to + // https://groups.google.com/forum/#!forum/grpc-io. + protected: + void set_byte_count(int64_t byte_count) { byte_count_ = byte_count; } + int64_t backup_count() { return backup_count_; } + void set_backup_count(int64_t backup_count) { backup_count_ = backup_count; } + grpc_byte_buffer_reader* reader() { return &reader_; } + grpc_slice* slice() { return &slice_; } + + private: + int64_t byte_count_; ///< total bytes read since object creation + int64_t backup_count_; ///< how far backed up in the stream we are + grpc_byte_buffer_reader reader_; ///< internal object to read \a grpc_slice + ///< from the \a grpc_byte_buffer + grpc_slice slice_; ///< current slice passed back to the caller + Status status_; ///< status of the entire object +}; + +} // namespace grpc + +#endif // GRPCPP_IMPL_CODEGEN_PROTO_BUFFER_READER_H diff --git a/include/grpcpp/impl/codegen/proto_buffer_writer.h b/include/grpcpp/impl/codegen/proto_buffer_writer.h new file mode 100644 index 0000000000..faf99800cd --- /dev/null +++ b/include/grpcpp/impl/codegen/proto_buffer_writer.h @@ -0,0 +1,168 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCPP_IMPL_CODEGEN_PROTO_BUFFER_WRITER_H +#define GRPCPP_IMPL_CODEGEN_PROTO_BUFFER_WRITER_H + +#include <type_traits> + +#include <grpc/impl/codegen/grpc_types.h> +#include <grpc/impl/codegen/slice.h> +#include <grpcpp/impl/codegen/byte_buffer.h> +#include <grpcpp/impl/codegen/config_protobuf.h> +#include <grpcpp/impl/codegen/core_codegen_interface.h> +#include <grpcpp/impl/codegen/serialization_traits.h> +#include <grpcpp/impl/codegen/status.h> + +/// This header provides an object that writes bytes directly into a +/// grpc::ByteBuffer, via the ZeroCopyOutputStream interface + +namespace grpc { + +extern CoreCodegenInterface* g_core_codegen_interface; + +// Forward declaration for testing use only +namespace internal { +class GrpcProtoBufferWriterPeer; +} // namespace internal + +const int kGrpcProtoBufferWriterMaxBufferLength = 1024 * 1024; + +/// This is a specialization of the protobuf class ZeroCopyOutputStream. +/// The principle is to give the proto layer one buffer of bytes at a time +/// that it can use to serialize the next portion of the message, with the +/// option to "backup" if more buffer is given than required at the last buffer. +/// +/// Read more about ZeroCopyOutputStream interface here: +/// https://developers.google.com/protocol-buffers/docs/reference/cpp/google.protobuf.io.zero_copy_stream#ZeroCopyOutputStream +class GrpcProtoBufferWriter + : public ::grpc::protobuf::io::ZeroCopyOutputStream { + public: + /// Constructor for this derived class + /// + /// \param[out] byte_buffer A pointer to the grpc::ByteBuffer created + /// \param block_size How big are the chunks to allocate at a time + /// \param total_size How many total bytes are required for this proto + GrpcProtoBufferWriter(ByteBuffer* byte_buffer, int block_size, int total_size) + : block_size_(block_size), + total_size_(total_size), + byte_count_(0), + have_backup_(false) { + GPR_CODEGEN_ASSERT(!byte_buffer->Valid()); + /// Create an empty raw byte buffer and look at its underlying slice buffer + grpc_byte_buffer* bp = + g_core_codegen_interface->grpc_raw_byte_buffer_create(NULL, 0); + byte_buffer->set_buffer(bp); + slice_buffer_ = &bp->data.raw.slice_buffer; + } + + ~GrpcProtoBufferWriter() { + if (have_backup_) { + g_core_codegen_interface->grpc_slice_unref(backup_slice_); + } + } + + /// Give the proto library the next buffer of bytes and its size. It is + /// safe for the caller to write from data[0, size - 1]. + bool Next(void** data, int* size) override { + // Protobuf should not ask for more memory than total_size_. + GPR_CODEGEN_ASSERT(byte_count_ < total_size_); + // 1. Use the remaining backup slice if we have one + // 2. Otherwise allocate a slice, up to the remaining length needed + // or our maximum allocation size + // 3. Provide the slice start and size available + // 4. Add the slice being returned to the slice buffer + size_t remain = total_size_ - byte_count_; + if (have_backup_) { + /// If we have a backup slice, we should use it first + slice_ = backup_slice_; + have_backup_ = false; + if (GRPC_SLICE_LENGTH(slice_) > remain) { + GRPC_SLICE_SET_LENGTH(slice_, remain); + } + } else { + // When less than a whole block is needed, only allocate that much. + // But make sure the allocated slice is not inlined. + size_t allocate_length = + remain > static_cast<size_t>(block_size_) ? block_size_ : remain; + slice_ = g_core_codegen_interface->grpc_slice_malloc( + allocate_length > GRPC_SLICE_INLINED_SIZE + ? allocate_length + : GRPC_SLICE_INLINED_SIZE + 1); + } + *data = GRPC_SLICE_START_PTR(slice_); + // On win x64, int is only 32bit + GPR_CODEGEN_ASSERT(GRPC_SLICE_LENGTH(slice_) <= INT_MAX); + byte_count_ += * size = (int)GRPC_SLICE_LENGTH(slice_); + g_core_codegen_interface->grpc_slice_buffer_add(slice_buffer_, slice_); + return true; + } + + /// Backup by \a count bytes because Next returned more bytes than needed + /// (only used in the last buffer). \a count must be less than or equal too + /// the last buffer returned from next. + void BackUp(int count) override { + /// 1. Remove the partially-used last slice from the slice buffer + /// 2. Split it into the needed (if any) and unneeded part + /// 3. Add the needed part back to the slice buffer + /// 4. Mark that we still have the remaining part (for later use/unref) + GPR_CODEGEN_ASSERT(count <= static_cast<int>(GRPC_SLICE_LENGTH(slice_))); + g_core_codegen_interface->grpc_slice_buffer_pop(slice_buffer_); + if ((size_t)count == GRPC_SLICE_LENGTH(slice_)) { + backup_slice_ = slice_; + } else { + backup_slice_ = g_core_codegen_interface->grpc_slice_split_tail( + &slice_, GRPC_SLICE_LENGTH(slice_) - count); + g_core_codegen_interface->grpc_slice_buffer_add(slice_buffer_, slice_); + } + // It's dangerous to keep an inlined grpc_slice as the backup slice, since + // on a following Next() call, a reference will be returned to this slice + // via GRPC_SLICE_START_PTR, which will not be an address held by + // slice_buffer_. + have_backup_ = backup_slice_.refcount != NULL; + byte_count_ -= count; + } + + /// Returns the total number of bytes written since this object was created. + grpc::protobuf::int64 ByteCount() const override { return byte_count_; } + + // These protected members are needed to support internal optimizations. + // they expose internal bits of grpc core that are NOT stable. If you have + // a use case needs to use one of these functions, please send an email to + // https://groups.google.com/forum/#!forum/grpc-io. + protected: + grpc_slice_buffer* slice_buffer() { return slice_buffer_; } + void set_byte_count(int64_t byte_count) { byte_count_ = byte_count; } + + private: + // friend for testing purposes only + friend class internal::GrpcProtoBufferWriterPeer; + const int block_size_; ///< size to alloc for each new \a grpc_slice needed + const int total_size_; ///< byte size of proto being serialized + int64_t byte_count_; ///< bytes written since this object was created + grpc_slice_buffer* + slice_buffer_; ///< internal buffer of slices holding the serialized data + bool have_backup_; ///< if we are holding a backup slice or not + grpc_slice backup_slice_; ///< holds space we can still write to, if the + ///< caller has called BackUp + grpc_slice slice_; ///< current slice passed back to the caller +}; + +} // namespace grpc + +#endif // GRPCPP_IMPL_CODEGEN_PROTO_BUFFER_WRITER_H diff --git a/include/grpcpp/impl/codegen/proto_utils.h b/include/grpcpp/impl/codegen/proto_utils.h index 81438ee1d5..ec1dccaa74 100644 --- a/include/grpcpp/impl/codegen/proto_utils.h +++ b/include/grpcpp/impl/codegen/proto_utils.h @@ -24,203 +24,62 @@ #include <grpc/impl/codegen/byte_buffer_reader.h> #include <grpc/impl/codegen/grpc_types.h> #include <grpc/impl/codegen/slice.h> +#include <grpcpp/impl/codegen/byte_buffer.h> #include <grpcpp/impl/codegen/config_protobuf.h> #include <grpcpp/impl/codegen/core_codegen_interface.h> +#include <grpcpp/impl/codegen/proto_buffer_reader.h> +#include <grpcpp/impl/codegen/proto_buffer_writer.h> #include <grpcpp/impl/codegen/serialization_traits.h> +#include <grpcpp/impl/codegen/slice.h> #include <grpcpp/impl/codegen/status.h> +/// This header provides serialization and deserialization between gRPC +/// messages serialized using protobuf and the C++ objects they represent. + namespace grpc { extern CoreCodegenInterface* g_core_codegen_interface; -namespace internal { - -class GrpcBufferWriterPeer; - -const int kGrpcBufferWriterMaxBufferLength = 1024 * 1024; - -class GrpcBufferWriter : public ::grpc::protobuf::io::ZeroCopyOutputStream { - public: - GrpcBufferWriter(grpc_byte_buffer** bp, int block_size, int total_size) - : block_size_(block_size), - total_size_(total_size), - byte_count_(0), - have_backup_(false) { - *bp = g_core_codegen_interface->grpc_raw_byte_buffer_create(NULL, 0); - slice_buffer_ = &(*bp)->data.raw.slice_buffer; - } - - ~GrpcBufferWriter() override { - if (have_backup_) { - g_core_codegen_interface->grpc_slice_unref(backup_slice_); - } - } - - bool Next(void** data, int* size) override { - // Protobuf should not ask for more memory than total_size_. - GPR_CODEGEN_ASSERT(byte_count_ < total_size_); - size_t remain = total_size_ - byte_count_; - if (have_backup_) { - slice_ = backup_slice_; - have_backup_ = false; - if (GRPC_SLICE_LENGTH(slice_) > remain) { - GRPC_SLICE_SET_LENGTH(slice_, remain); - } - } else { - // When less than a whole block is needed, only allocate that much. - // But make sure the allocated slice is not inlined. - size_t allocate_length = - remain > static_cast<size_t>(block_size_) ? block_size_ : remain; - slice_ = g_core_codegen_interface->grpc_slice_malloc( - allocate_length > GRPC_SLICE_INLINED_SIZE - ? allocate_length - : GRPC_SLICE_INLINED_SIZE + 1); - } - *data = GRPC_SLICE_START_PTR(slice_); - // On win x64, int is only 32bit - GPR_CODEGEN_ASSERT(GRPC_SLICE_LENGTH(slice_) <= INT_MAX); - byte_count_ += * size = (int)GRPC_SLICE_LENGTH(slice_); - g_core_codegen_interface->grpc_slice_buffer_add(slice_buffer_, slice_); - return true; - } - - void BackUp(int count) override { - g_core_codegen_interface->grpc_slice_buffer_pop(slice_buffer_); - if ((size_t)count == GRPC_SLICE_LENGTH(slice_)) { - backup_slice_ = slice_; - } else { - backup_slice_ = g_core_codegen_interface->grpc_slice_split_tail( - &slice_, GRPC_SLICE_LENGTH(slice_) - count); - g_core_codegen_interface->grpc_slice_buffer_add(slice_buffer_, slice_); - } - // It's dangerous to keep an inlined grpc_slice as the backup slice, since - // on a following Next() call, a reference will be returned to this slice - // via GRPC_SLICE_START_PTR, which will not be an adddress held by - // slice_buffer_. - have_backup_ = backup_slice_.refcount != NULL; - byte_count_ -= count; - } - - grpc::protobuf::int64 ByteCount() const override { return byte_count_; } - - protected: - friend class GrpcBufferWriterPeer; - const int block_size_; - const int total_size_; - int64_t byte_count_; - grpc_slice_buffer* slice_buffer_; - bool have_backup_; - grpc_slice backup_slice_; - grpc_slice slice_; -}; - -class GrpcBufferReader : public ::grpc::protobuf::io::ZeroCopyInputStream { - public: - explicit GrpcBufferReader(grpc_byte_buffer* buffer) - : byte_count_(0), backup_count_(0), status_() { - if (!g_core_codegen_interface->grpc_byte_buffer_reader_init(&reader_, - buffer)) { - status_ = Status(StatusCode::INTERNAL, - "Couldn't initialize byte buffer reader"); - } - } - ~GrpcBufferReader() override { - g_core_codegen_interface->grpc_byte_buffer_reader_destroy(&reader_); - } - - bool Next(const void** data, int* size) override { - if (!status_.ok()) { - return false; - } - if (backup_count_ > 0) { - *data = GRPC_SLICE_START_PTR(slice_) + GRPC_SLICE_LENGTH(slice_) - - backup_count_; - GPR_CODEGEN_ASSERT(backup_count_ <= INT_MAX); - *size = (int)backup_count_; - backup_count_ = 0; - return true; - } - if (!g_core_codegen_interface->grpc_byte_buffer_reader_next(&reader_, - &slice_)) { - return false; - } - g_core_codegen_interface->grpc_slice_unref(slice_); - *data = GRPC_SLICE_START_PTR(slice_); - // On win x64, int is only 32bit - GPR_CODEGEN_ASSERT(GRPC_SLICE_LENGTH(slice_) <= INT_MAX); - byte_count_ += * size = (int)GRPC_SLICE_LENGTH(slice_); - return true; - } - - Status status() const { return status_; } - - void BackUp(int count) override { backup_count_ = count; } - - bool Skip(int count) override { - const void* data; - int size; - while (Next(&data, &size)) { - if (size >= count) { - BackUp(size - count); - return true; - } - // size < count; - count -= size; - } - // error or we have too large count; - return false; - } - - grpc::protobuf::int64 ByteCount() const override { - return byte_count_ - backup_count_; - } - - protected: - int64_t byte_count_; - int64_t backup_count_; - grpc_byte_buffer_reader reader_; - grpc_slice slice_; - Status status_; -}; - -// BufferWriter must be a subclass of io::ZeroCopyOutputStream. -template <class BufferWriter, class T> -Status GenericSerialize(const grpc::protobuf::Message& msg, - grpc_byte_buffer** bp, bool* own_buffer) { - static_assert( - std::is_base_of<protobuf::io::ZeroCopyOutputStream, BufferWriter>::value, - "BufferWriter must be a subclass of io::ZeroCopyOutputStream"); +// ProtoBufferWriter must be a subclass of ::protobuf::io::ZeroCopyOutputStream. +template <class ProtoBufferWriter, class T> +Status GenericSerialize(const grpc::protobuf::Message& msg, ByteBuffer* bb, + bool* own_buffer) { + static_assert(std::is_base_of<protobuf::io::ZeroCopyOutputStream, + ProtoBufferWriter>::value, + "ProtoBufferWriter must be a subclass of " + "::protobuf::io::ZeroCopyOutputStream"); *own_buffer = true; int byte_size = msg.ByteSize(); if ((size_t)byte_size <= GRPC_SLICE_INLINED_SIZE) { - grpc_slice slice = g_core_codegen_interface->grpc_slice_malloc(byte_size); - GPR_CODEGEN_ASSERT( - GRPC_SLICE_END_PTR(slice) == - msg.SerializeWithCachedSizesToArray(GRPC_SLICE_START_PTR(slice))); - *bp = g_core_codegen_interface->grpc_raw_byte_buffer_create(&slice, 1); - g_core_codegen_interface->grpc_slice_unref(slice); + Slice slice(byte_size); + // We serialize directly into the allocated slices memory + GPR_CODEGEN_ASSERT(slice.end() == msg.SerializeWithCachedSizesToArray( + const_cast<uint8_t*>(slice.begin()))); + ByteBuffer tmp(&slice, 1); + bb->Swap(&tmp); return g_core_codegen_interface->ok(); } - BufferWriter writer(bp, kGrpcBufferWriterMaxBufferLength, byte_size); + ProtoBufferWriter writer(bb, kGrpcProtoBufferWriterMaxBufferLength, + byte_size); return msg.SerializeToZeroCopyStream(&writer) ? g_core_codegen_interface->ok() : Status(StatusCode::INTERNAL, "Failed to serialize message"); } -// BufferReader must be a subclass of io::ZeroCopyInputStream. -template <class BufferReader, class T> -Status GenericDeserialize(grpc_byte_buffer* buffer, - grpc::protobuf::Message* msg) { - static_assert( - std::is_base_of<protobuf::io::ZeroCopyInputStream, BufferReader>::value, - "BufferReader must be a subclass of io::ZeroCopyInputStream"); +// BufferReader must be a subclass of ::protobuf::io::ZeroCopyInputStream. +template <class ProtoBufferReader, class T> +Status GenericDeserialize(ByteBuffer* buffer, grpc::protobuf::Message* msg) { + static_assert(std::is_base_of<protobuf::io::ZeroCopyInputStream, + ProtoBufferReader>::value, + "ProtoBufferReader must be a subclass of " + "::protobuf::io::ZeroCopyInputStream"); if (buffer == nullptr) { return Status(StatusCode::INTERNAL, "No payload"); } Status result = g_core_codegen_interface->ok(); { - BufferReader reader(buffer); + ProtoBufferReader reader(buffer); if (!reader.status().ok()) { return reader.status(); } @@ -233,12 +92,10 @@ Status GenericDeserialize(grpc_byte_buffer* buffer, result = Status(StatusCode::INTERNAL, "Did not read entire message"); } } - g_core_codegen_interface->grpc_byte_buffer_destroy(buffer); + buffer->Clear(); return result; } -} // namespace internal - // this is needed so the following class does not conflict with protobuf // serializers that utilize internal-only tools. #ifdef GRPC_OPEN_SOURCE_PROTO @@ -249,16 +106,13 @@ template <class T> class SerializationTraits<T, typename std::enable_if<std::is_base_of< grpc::protobuf::Message, T>::value>::type> { public: - static Status Serialize(const grpc::protobuf::Message& msg, - grpc_byte_buffer** bp, bool* own_buffer) { - return internal::GenericSerialize<internal::GrpcBufferWriter, T>( - msg, bp, own_buffer); + static Status Serialize(const grpc::protobuf::Message& msg, ByteBuffer* bb, + bool* own_buffer) { + return GenericSerialize<GrpcProtoBufferWriter, T>(msg, bb, own_buffer); } - static Status Deserialize(grpc_byte_buffer* buffer, - grpc::protobuf::Message* msg) { - return internal::GenericDeserialize<internal::GrpcBufferReader, T>(buffer, - msg); + static Status Deserialize(ByteBuffer* buffer, grpc::protobuf::Message* msg) { + return GenericDeserialize<GrpcProtoBufferReader, T>(buffer, msg); } }; #endif diff --git a/include/grpcpp/impl/codegen/server_context.h b/include/grpcpp/impl/codegen/server_context.h index bb357d6734..bced4202dd 100644 --- a/include/grpcpp/impl/codegen/server_context.h +++ b/include/grpcpp/impl/codegen/server_context.h @@ -88,8 +88,8 @@ class ServerContextTestSpouse; /// /// Context settings are only relevant to the call handler they are supplied to, /// that is to say, they aren't sticky across multiple calls. Some of these -/// settings, such as the compression options, can be made persistant at server -/// construction time by specifying the approriate \a ChannelArguments +/// settings, such as the compression options, can be made persistent at server +/// construction time by specifying the appropriate \a ChannelArguments /// to a \a grpc::ServerBuilder, via \a ServerBuilder::AddChannelArgument. /// /// \warning ServerContext instances should \em not be reused across rpcs. diff --git a/include/grpcpp/impl/codegen/slice.h b/include/grpcpp/impl/codegen/slice.h index fcccd4b68e..8966559dc8 100644 --- a/include/grpcpp/impl/codegen/slice.h +++ b/include/grpcpp/impl/codegen/slice.h @@ -35,34 +35,43 @@ namespace grpc { class Slice final { public: /// Construct an empty slice. - Slice(); + Slice() : slice_(g_core_codegen_interface->grpc_empty_slice()) {} /// Destructor - drops one reference. - ~Slice(); + ~Slice() { g_core_codegen_interface->grpc_slice_unref(slice_); } enum AddRef { ADD_REF }; /// Construct a slice from \a slice, adding a reference. - Slice(grpc_slice slice, AddRef); + Slice(grpc_slice slice, AddRef) + : slice_(g_core_codegen_interface->grpc_slice_ref(slice)) {} enum StealRef { STEAL_REF }; /// Construct a slice from \a slice, stealing a reference. - Slice(grpc_slice slice, StealRef); + Slice(grpc_slice slice, StealRef) : slice_(slice) {} /// Allocate a slice of specified size - Slice(size_t len); + Slice(size_t len) + : slice_(g_core_codegen_interface->grpc_slice_malloc(len)) {} /// Construct a slice from a copied buffer - Slice(const void* buf, size_t len); + Slice(const void* buf, size_t len) + : slice_(g_core_codegen_interface->grpc_slice_from_copied_buffer( + reinterpret_cast<const char*>(buf), len)) {} /// Construct a slice from a copied string - Slice(const grpc::string& str); + Slice(const grpc::string& str) + : slice_(g_core_codegen_interface->grpc_slice_from_copied_buffer( + str.c_str(), str.length())) {} enum StaticSlice { STATIC_SLICE }; /// Construct a slice from a static buffer - Slice(const void* buf, size_t len, StaticSlice); + Slice(const void* buf, size_t len, StaticSlice) + : slice_(g_core_codegen_interface->grpc_slice_from_static_buffer( + reinterpret_cast<const char*>(buf), len)) {} /// Copy constructor, adds a reference. - Slice(const Slice& other); + Slice(const Slice& other) + : slice_(g_core_codegen_interface->grpc_slice_ref(other.slice_)) {} /// Assignment, reference count is unchanged. Slice& operator=(Slice other) { @@ -75,14 +84,18 @@ class Slice final { /// user data pointer passed in at destruction. Can be the same as buf or /// different (e.g., if data is part of a larger structure that must be /// destroyed when the data is no longer needed) - Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data); + Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data) + : slice_(g_core_codegen_interface->grpc_slice_new_with_user_data( + buf, len, destroy, user_data)) {} /// Specialization of above for common case where buf == user_data Slice(void* buf, size_t len, void (*destroy)(void*)) : Slice(buf, len, destroy, buf) {} /// Similar to the above but has a destroy that also takes slice length - Slice(void* buf, size_t len, void (*destroy)(void*, size_t)); + Slice(void* buf, size_t len, void (*destroy)(void*, size_t)) + : slice_(g_core_codegen_interface->grpc_slice_new_with_len(buf, len, + destroy)) {} /// Byte size. size_t size() const { return GRPC_SLICE_LENGTH(slice_); } @@ -94,7 +107,9 @@ class Slice final { const uint8_t* end() const { return GRPC_SLICE_END_PTR(slice_); } /// Raw C slice. Caller needs to call grpc_slice_unref when done. - grpc_slice c_slice() const; + grpc_slice c_slice() const { + return g_core_codegen_interface->grpc_slice_ref(slice_); + } private: friend class ByteBuffer; diff --git a/include/grpcpp/support/proto_buffer_reader.h b/include/grpcpp/support/proto_buffer_reader.h new file mode 100644 index 0000000000..4cdb65d531 --- /dev/null +++ b/include/grpcpp/support/proto_buffer_reader.h @@ -0,0 +1,24 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCPP_SUPPORT_PROTO_BUFFER_READER_H +#define GRPCPP_SUPPORT_PROTO_BUFFER_READER_H + +#include <grpcpp/impl/codegen/proto_buffer_reader.h> + +#endif // GRPCPP_SUPPORT_PROTO_BUFFER_READER_H diff --git a/include/grpcpp/support/proto_buffer_writer.h b/include/grpcpp/support/proto_buffer_writer.h new file mode 100644 index 0000000000..01cf29c457 --- /dev/null +++ b/include/grpcpp/support/proto_buffer_writer.h @@ -0,0 +1,24 @@ +/* + * + * Copyright 2018 gRPC authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +#ifndef GRPCPP_SUPPORT_PROTO_BUFFER_WRITER_H +#define GRPCPP_SUPPORT_PROTO_BUFFER_WRITER_H + +#include <grpcpp/impl/codegen/proto_buffer_writer.h> + +#endif // GRPCPP_SUPPORT_PROTO_BUFFER_WRITER_H diff --git a/package.xml b/package.xml index 7213d1645f..b9cb22b647 100644 --- a/package.xml +++ b/package.xml @@ -13,8 +13,8 @@ <date>2018-01-19</date> <time>16:06:07</time> <version> - <release>1.11.0dev</release> - <api>1.11.0dev</api> + <release>1.12.0dev</release> + <api>1.12.0dev</api> </version> <stability> <release>beta</release> diff --git a/src/compiler/objective_c_generator.cc b/src/compiler/objective_c_generator.cc index ffdeb8f6b0..39f68cb956 100644 --- a/src/compiler/objective_c_generator.cc +++ b/src/compiler/objective_c_generator.cc @@ -206,7 +206,7 @@ void PrintMethodImplementations(Printer* printer, } } for (auto one_class : classes) { - output += " @class " + one_class + ";\n"; + output += "@class " + one_class + ";\n"; } return output; diff --git a/src/compiler/objective_c_plugin.cc b/src/compiler/objective_c_plugin.cc index 76703d79cd..f0fe3688cc 100644 --- a/src/compiler/objective_c_plugin.cc +++ b/src/compiler/objective_c_plugin.cc @@ -118,11 +118,11 @@ class ObjectiveCGrpcGenerator : public grpc::protobuf::compiler::CodeGenerator { Write(context, file_name + ".pbrpc.h", PreprocIfNot(kForwardDeclare, imports) + "\n" + PreprocIfNot(kProtocolOnly, system_imports) + "\n" + - PreprocIfElse(kForwardDeclare, class_declarations, - class_imports) + - "\n" + forward_declarations + "\n" + kNonNullBegin + "\n" + - protocols + "\n" + PreprocIfNot(kProtocolOnly, interfaces) + - "\n" + kNonNullEnd + "\n"); + class_declarations + "\n" + + PreprocIfNot(kForwardDeclare, class_imports) + "\n" + + forward_declarations + "\n" + kNonNullBegin + "\n" + protocols + + "\n" + PreprocIfNot(kProtocolOnly, interfaces) + "\n" + + kNonNullEnd + "\n"); } { diff --git a/src/core/lib/debug/trace.cc b/src/core/lib/debug/trace.cc index b0e0f2ba7c..01c1e867d9 100644 --- a/src/core/lib/debug/trace.cc +++ b/src/core/lib/debug/trace.cc @@ -55,7 +55,8 @@ bool TraceFlagList::Set(const char* name, bool enabled) { found = true; } } - if (!found) { + // check for unknowns, but ignore "", to allow to GRPC_TRACE= + if (!found && 0 != strcmp(name, "")) { gpr_log(GPR_ERROR, "Unknown trace var: '%s'", name); return false; /* early return */ } diff --git a/src/core/lib/iomgr/ev_epollex_linux.cc b/src/core/lib/iomgr/ev_epollex_linux.cc index 0ef7c03056..44d8cf2b1e 100644 --- a/src/core/lib/iomgr/ev_epollex_linux.cc +++ b/src/core/lib/iomgr/ev_epollex_linux.cc @@ -59,7 +59,10 @@ //#define GRPC_EPOLLEX_CREATE_WORKERS_ON_HEAP 1 #define MAX_EPOLL_EVENTS 100 -#define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 1 +// TODO(juanlishen): We use a greater-than-one value here as a workaround fix to +// a keepalive ping timeout issue. We may want to revert https://github +// .com/grpc/grpc/pull/14943 once we figure out the root cause. +#define MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL 16 grpc_core::DebugOnlyTraceFlag grpc_trace_pollable_refcount(false, "pollable_refcount"); @@ -198,6 +201,7 @@ struct grpc_pollset_worker { struct grpc_pollset { gpr_mu mu; + gpr_atm worker_count; pollable* active_pollable; bool kicked_without_poller; grpc_closure* shutdown_closure; @@ -685,6 +689,7 @@ static grpc_error* pollset_kick_all(grpc_pollset* pollset) { static void pollset_init(grpc_pollset* pollset, gpr_mu** mu) { gpr_mu_init(&pollset->mu); + gpr_atm_no_barrier_store(&pollset->worker_count, 0); pollset->active_pollable = POLLABLE_REF(g_empty_pollable, "pollset"); pollset->kicked_without_poller = false; pollset->shutdown_closure = nullptr; @@ -758,8 +763,20 @@ static grpc_error* pollable_process_events(grpc_pollset* pollset, pollable* pollable_obj, bool drain) { GPR_TIMER_SCOPE("pollable_process_events", 0); static const char* err_desc = "pollset_process_events"; + // Use a simple heuristic to determine how many fd events to process + // per loop iteration. (events/workers) + int handle_count = 1; + int worker_count = gpr_atm_no_barrier_load(&pollset->worker_count); + GPR_ASSERT(worker_count > 0); + handle_count = + (pollable_obj->event_count - pollable_obj->event_cursor) / worker_count; + if (handle_count == 0) { + handle_count = 1; + } else if (handle_count > MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL) { + handle_count = MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL; + } grpc_error* error = GRPC_ERROR_NONE; - for (int i = 0; (drain || i < MAX_EPOLL_EVENTS_HANDLED_EACH_POLL_CALL) && + for (int i = 0; (drain || i < handle_count) && pollable_obj->event_cursor != pollable_obj->event_count; i++) { int n = pollable_obj->event_cursor++; @@ -884,6 +901,7 @@ static bool begin_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, GPR_TIMER_SCOPE("begin_worker", 0); bool do_poll = (pollset->shutdown_closure == nullptr && !pollset->already_shutdown); + gpr_atm_no_barrier_fetch_add(&pollset->worker_count, 1); if (worker_hdl != nullptr) *worker_hdl = worker; worker->initialized_cv = false; worker->kicked = false; @@ -964,6 +982,7 @@ static void end_worker(grpc_pollset* pollset, grpc_pollset_worker* worker, if (worker->initialized_cv) { gpr_cv_destroy(&worker->cv); } + gpr_atm_no_barrier_fetch_add(&pollset->worker_count, -1); } #ifndef NDEBUG diff --git a/src/core/lib/iomgr/ev_poll_posix.cc b/src/core/lib/iomgr/ev_poll_posix.cc index 2e375b4022..d9aba9b6a3 100644 --- a/src/core/lib/iomgr/ev_poll_posix.cc +++ b/src/core/lib/iomgr/ev_poll_posix.cc @@ -1530,6 +1530,12 @@ static void run_poll(void* args) { // This function overrides poll() to handle condition variable wakeup fds static int cvfd_poll(struct pollfd* fds, nfds_t nfds, int timeout) { + if (timeout == 0) { + // Don't bother using background threads for polling if timeout is 0, + // poll-cv might not wait for a poll to return otherwise. + // https://github.com/grpc/grpc/issues/13298 + return poll(fds, nfds, 0); + } unsigned int i; int res, idx; grpc_cv_node* pollcv; diff --git a/src/core/lib/iomgr/socket_utils_linux.cc b/src/core/lib/iomgr/socket_utils_linux.cc index b0207578de..34f93cc4b0 100644 --- a/src/core/lib/iomgr/socket_utils_linux.cc +++ b/src/core/lib/iomgr/socket_utils_linux.cc @@ -33,7 +33,6 @@ int grpc_accept4(int sockfd, grpc_resolved_address* resolved_addr, int nonblock, int cloexec) { int flags = 0; - GPR_ASSERT(sizeof(socklen_t) <= sizeof(size_t)); flags |= nonblock ? SOCK_NONBLOCK : 0; flags |= cloexec ? SOCK_CLOEXEC : 0; return accept4(sockfd, reinterpret_cast<grpc_sockaddr*>(resolved_addr->addr), diff --git a/src/core/lib/iomgr/socket_utils_posix.cc b/src/core/lib/iomgr/socket_utils_posix.cc index 2a49583ac4..c48da52ffb 100644 --- a/src/core/lib/iomgr/socket_utils_posix.cc +++ b/src/core/lib/iomgr/socket_utils_posix.cc @@ -34,9 +34,8 @@ int grpc_accept4(int sockfd, grpc_resolved_address* resolved_addr, int nonblock, int cloexec) { int fd, flags; - GPR_ASSERT(sizeof(socklen_t) <= sizeof(size_t)); - fd = accept(sockfd, (grpc_sockaddr*)resolved_addr->addr, - (socklen_t*)&resolved_addr->len); + fd = accept(sockfd, reinterpret_cast<grpc_sockaddr*>(resolved_addr->addr), + &resolved_addr->len); if (fd >= 0) { if (nonblock) { flags = fcntl(fd, F_GETFL, 0); diff --git a/src/core/lib/iomgr/socket_utils_uv.cc b/src/core/lib/iomgr/socket_utils_uv.cc index 8538abc7e4..7eba40c46b 100644 --- a/src/core/lib/iomgr/socket_utils_uv.cc +++ b/src/core/lib/iomgr/socket_utils_uv.cc @@ -38,8 +38,8 @@ int grpc_inet_pton(int af, const char* src, void* dst) { } const char* grpc_inet_ntop(int af, const void* src, char* dst, size_t size) { - /* Windows InetNtopA wants a mutable ip pointer */ - return inet_ntop(af, src, dst, (socklen_t)size); + uv_inet_ntop(af, src, dst, size); + return dst; } #endif /* GRPC_UV */ diff --git a/src/core/lib/iomgr/tcp_custom.h b/src/core/lib/iomgr/tcp_custom.h index 22caa149f8..784ef84222 100644 --- a/src/core/lib/iomgr/tcp_custom.h +++ b/src/core/lib/iomgr/tcp_custom.h @@ -62,8 +62,6 @@ typedef struct grpc_socket_vtable { const grpc_sockaddr* addr, int* len); grpc_error* (*getsockname)(grpc_custom_socket* socket, const grpc_sockaddr* addr, int* len); - grpc_error* (*setsockopt)(grpc_custom_socket* socket, int level, int optname, - const void* optval, uint32_t optlen); grpc_error* (*bind)(grpc_custom_socket* socket, const grpc_sockaddr* addr, size_t len, int flags); grpc_error* (*listen)(grpc_custom_socket* socket); diff --git a/src/core/lib/iomgr/tcp_server_custom.cc b/src/core/lib/iomgr/tcp_server_custom.cc index be92e61b62..79ba5c39ee 100644 --- a/src/core/lib/iomgr/tcp_server_custom.cc +++ b/src/core/lib/iomgr/tcp_server_custom.cc @@ -393,13 +393,6 @@ static grpc_error* tcp_server_add_port(grpc_tcp_server* s, grpc_custom_socket_vtable->init(socket, family); if (error == GRPC_ERROR_NONE) { -#if defined(GPR_LINUX) && defined(SO_REUSEPORT) - if (family == AF_INET || family == AF_INET6) { - int enable = 1; - grpc_custom_socket_vtable->setsockopt(socket, SOL_SOCKET, SO_REUSEPORT, - &enable, sizeof(enable)); - } -#endif /* GPR_LINUX && SO_REUSEPORT */ error = add_socket_to_server(s, socket, addr, port_index, &sp); } gpr_free(allocated_addr); diff --git a/src/core/lib/iomgr/tcp_server_windows.cc b/src/core/lib/iomgr/tcp_server_windows.cc index 77f3811dca..b01afdcc9d 100644 --- a/src/core/lib/iomgr/tcp_server_windows.cc +++ b/src/core/lib/iomgr/tcp_server_windows.cc @@ -129,6 +129,7 @@ static void destroy_server(void* arg, grpc_error* error) { gpr_free(sp); } grpc_channel_args_destroy(s->channel_args); + gpr_mu_destroy(&s->mu); gpr_free(s); } diff --git a/src/core/lib/iomgr/tcp_uv.cc b/src/core/lib/iomgr/tcp_uv.cc index 5e3166926b..f20f8dcb74 100644 --- a/src/core/lib/iomgr/tcp_uv.cc +++ b/src/core/lib/iomgr/tcp_uv.cc @@ -192,6 +192,15 @@ static grpc_error* uv_socket_init_helper(uv_socket_t* uv_socket, int domain) { if (status != 0) { return tcp_error_create("Failed to initialize UV tcp handle", status); } +#if defined(GPR_LINUX) && defined(SO_REUSEPORT) + if (domain == AF_INET || domain == AF_INET6) { + int enable = 1; + int fd; + uv_fileno((uv_handle_t*)tcp, &fd); + // TODO Handle error here. + setsockopt(fd, SOL_SOCKET, SO_REUSEPORT, &enable, sizeof(enable)); + } +#endif uv_socket->write_buffers = nullptr; uv_socket->read_len = 0; uv_tcp_nodelay(uv_socket->handle, 1); @@ -299,17 +308,6 @@ static grpc_error* uv_socket_listen(grpc_custom_socket* socket) { return tcp_error_create("Failed to listen to port", status); } -static grpc_error* uv_socket_setsockopt(grpc_custom_socket* socket, int level, - int option_name, const void* optval, - socklen_t option_len) { - int fd; - uv_socket_t* uv_socket = (uv_socket_t*)socket->impl; - uv_fileno((uv_handle_t*)uv_socket->handle, &fd); - // TODO Handle error here. Also, does this work on windows?? - setsockopt(fd, level, option_name, &optval, (socklen_t)option_len); - return GRPC_ERROR_NONE; -} - static void uv_tc_on_connect(uv_connect_t* req, int status) { grpc_custom_socket* socket = (grpc_custom_socket*)req->data; uv_socket_t* uv_socket = (uv_socket_t*)socket->impl; @@ -340,7 +338,6 @@ static void uv_socket_connect(grpc_custom_socket* socket, static grpc_resolved_addresses* handle_addrinfo_result( struct addrinfo* result) { struct addrinfo* resp; - struct addrinfo* prev; size_t i; grpc_resolved_addresses* addresses = (grpc_resolved_addresses*)gpr_malloc(sizeof(grpc_resolved_addresses)); @@ -350,16 +347,13 @@ static grpc_resolved_addresses* handle_addrinfo_result( } addresses->addrs = (grpc_resolved_address*)gpr_malloc( sizeof(grpc_resolved_address) * addresses->naddrs); - i = 0; - resp = result; - while (resp != nullptr) { + for (resp = result, i = 0; resp != nullptr; resp = resp->ai_next, i++) { memcpy(&addresses->addrs[i].addr, resp->ai_addr, resp->ai_addrlen); addresses->addrs[i].len = resp->ai_addrlen; - i++; - prev = resp; - resp = resp->ai_next; - gpr_free(prev); } + // addrinfo objects are allocated by libuv (e.g. in uv_getaddrinfo) + // and not by gpr_malloc + uv_freeaddrinfo(result); return addresses; } @@ -415,10 +409,9 @@ static void uv_resolve_async(grpc_custom_resolver* r, char* host, char* port) { grpc_custom_resolver_vtable uv_resolver_vtable = {uv_resolve, uv_resolve_async}; grpc_socket_vtable grpc_uv_socket_vtable = { - uv_socket_init, uv_socket_connect, uv_socket_destroy, - uv_socket_shutdown, uv_socket_close, uv_socket_write, - uv_socket_read, uv_socket_getpeername, uv_socket_getsockname, - uv_socket_setsockopt, uv_socket_bind, uv_socket_listen, - uv_socket_accept}; + uv_socket_init, uv_socket_connect, uv_socket_destroy, + uv_socket_shutdown, uv_socket_close, uv_socket_write, + uv_socket_read, uv_socket_getpeername, uv_socket_getsockname, + uv_socket_bind, uv_socket_listen, uv_socket_accept}; #endif diff --git a/src/core/lib/iomgr/timer_generic.cc b/src/core/lib/iomgr/timer_generic.cc index 93e654b7fa..0c6f236f83 100644 --- a/src/core/lib/iomgr/timer_generic.cc +++ b/src/core/lib/iomgr/timer_generic.cc @@ -97,6 +97,12 @@ static void init_timer_ht() { } } +static void destroy_timer_ht() { + for (int i = 0; i < NUM_HASH_BUCKETS; i++) { + gpr_mu_destroy(&g_hash_mu[i]); + } +} + static bool is_in_ht(grpc_timer* t) { size_t i = GPR_HASH_POINTER(t, NUM_HASH_BUCKETS); @@ -188,6 +194,7 @@ static void validate_non_pending_timer(grpc_timer* t) { } #define INIT_TIMER_HASH_TABLE() init_timer_ht() +#define DESTROY_TIMER_HASH_TABLE() destroy_timer_ht() #define ADD_TO_HASH_TABLE(t) add_to_ht((t)) #define REMOVE_FROM_HASH_TABLE(t) remove_from_ht((t)) #define VALIDATE_NON_PENDING_TIMER(t) validate_non_pending_timer((t)) @@ -195,6 +202,7 @@ static void validate_non_pending_timer(grpc_timer* t) { #else #define INIT_TIMER_HASH_TABLE() +#define DESTROY_TIMER_HASH_TABLE() #define ADD_TO_HASH_TABLE(t) #define REMOVE_FROM_HASH_TABLE(t) #define VALIDATE_NON_PENDING_TIMER(t) @@ -283,6 +291,8 @@ static void timer_list_shutdown() { gpr_free(g_shards); gpr_free(g_shard_queue); g_shared_mutables.initialized = false; + + DESTROY_TIMER_HASH_TABLE(); } /* returns true if the first element in the list */ diff --git a/src/core/lib/slice/slice.cc b/src/core/lib/slice/slice.cc index 585b41cf91..419474129b 100644 --- a/src/core/lib/slice/slice.cc +++ b/src/core/lib/slice/slice.cc @@ -69,8 +69,12 @@ grpc_slice grpc_slice_ref(grpc_slice slice) { /* Public API */ void grpc_slice_unref(grpc_slice slice) { - grpc_core::ExecCtx exec_ctx; - grpc_slice_unref_internal(slice); + if (grpc_core::ExecCtx::Get() == nullptr) { + grpc_core::ExecCtx exec_ctx; + grpc_slice_unref_internal(slice); + } else { + grpc_slice_unref_internal(slice); + } } /* grpc_slice_from_static_string support structure - a refcount that does diff --git a/src/core/lib/slice/slice_buffer.cc b/src/core/lib/slice/slice_buffer.cc index 58ca495517..fd56997388 100644 --- a/src/core/lib/slice/slice_buffer.cc +++ b/src/core/lib/slice/slice_buffer.cc @@ -75,8 +75,12 @@ void grpc_slice_buffer_destroy_internal(grpc_slice_buffer* sb) { } void grpc_slice_buffer_destroy(grpc_slice_buffer* sb) { - grpc_core::ExecCtx exec_ctx; - grpc_slice_buffer_destroy_internal(sb); + if (grpc_core::ExecCtx::Get() == nullptr) { + grpc_core::ExecCtx exec_ctx; + grpc_slice_buffer_destroy_internal(sb); + } else { + grpc_slice_buffer_destroy_internal(sb); + } } uint8_t* grpc_slice_buffer_tiny_add(grpc_slice_buffer* sb, size_t n) { @@ -176,8 +180,12 @@ void grpc_slice_buffer_reset_and_unref_internal(grpc_slice_buffer* sb) { } void grpc_slice_buffer_reset_and_unref(grpc_slice_buffer* sb) { - grpc_core::ExecCtx exec_ctx; - grpc_slice_buffer_reset_and_unref_internal(sb); + if (grpc_core::ExecCtx::Get() == nullptr) { + grpc_core::ExecCtx exec_ctx; + grpc_slice_buffer_reset_and_unref_internal(sb); + } else { + grpc_slice_buffer_reset_and_unref_internal(sb); + } } void grpc_slice_buffer_swap(grpc_slice_buffer* a, grpc_slice_buffer* b) { diff --git a/src/core/lib/surface/call.cc b/src/core/lib/surface/call.cc index c683cc02de..9a9113643d 100644 --- a/src/core/lib/surface/call.cc +++ b/src/core/lib/surface/call.cc @@ -878,8 +878,8 @@ static void set_encodings_accepted_by_peer(grpc_call* call, grpc_mdelem mdel, } else { char* accept_encoding_entry_str = grpc_slice_to_c_string(accept_encoding_entry_slice); - gpr_log(GPR_ERROR, - "Invalid entry in accept encoding metadata: '%s'. Ignoring.", + gpr_log(GPR_DEBUG, + "Unknown entry in accept encoding metadata: '%s'. Ignoring.", accept_encoding_entry_str); gpr_free(accept_encoding_entry_str); } diff --git a/src/core/lib/surface/version.cc b/src/core/lib/surface/version.cc index be196a78bc..a712e10037 100644 --- a/src/core/lib/surface/version.cc +++ b/src/core/lib/surface/version.cc @@ -25,4 +25,4 @@ const char* grpc_version_string(void) { return "6.0.0-dev"; } -const char* grpc_g_stands_for(void) { return "gorgeous"; } +const char* grpc_g_stands_for(void) { return "glorious"; } diff --git a/src/cpp/common/core_codegen.cc b/src/cpp/common/core_codegen.cc index aa9788da76..619aacadaa 100644 --- a/src/cpp/common/core_codegen.cc +++ b/src/cpp/common/core_codegen.cc @@ -98,6 +98,10 @@ void CoreCodegen::grpc_byte_buffer_destroy(grpc_byte_buffer* bb) { ::grpc_byte_buffer_destroy(bb); } +size_t CoreCodegen::grpc_byte_buffer_length(grpc_byte_buffer* bb) { + return ::grpc_byte_buffer_length(bb); +} + grpc_call_error CoreCodegen::grpc_call_cancel_with_status( grpc_call* call, grpc_status_code status, const char* description, void* reserved) { @@ -135,6 +139,12 @@ grpc_slice CoreCodegen::grpc_slice_new_with_user_data(void* p, size_t len, return ::grpc_slice_new_with_user_data(p, len, destroy, user_data); } +grpc_slice CoreCodegen::grpc_slice_new_with_len(void* p, size_t len, + void (*destroy)(void*, + size_t)) { + return ::grpc_slice_new_with_len(p, len, destroy); +} + grpc_slice CoreCodegen::grpc_empty_slice() { return ::grpc_empty_slice(); } grpc_slice CoreCodegen::grpc_slice_malloc(size_t length) { diff --git a/src/cpp/common/version_cc.cc b/src/cpp/common/version_cc.cc index fb1723c816..d669ea21a9 100644 --- a/src/cpp/common/version_cc.cc +++ b/src/cpp/common/version_cc.cc @@ -22,5 +22,5 @@ #include <grpcpp/grpcpp.h> namespace grpc { -grpc::string Version() { return "1.11.0-dev"; } +grpc::string Version() { return "1.12.0-dev"; } } // namespace grpc diff --git a/src/cpp/util/byte_buffer_cc.cc b/src/cpp/util/byte_buffer_cc.cc index fbc1768bcc..8700f96d8d 100644 --- a/src/cpp/util/byte_buffer_cc.cc +++ b/src/cpp/util/byte_buffer_cc.cc @@ -25,32 +25,6 @@ namespace grpc { static internal::GrpcLibraryInitializer g_gli_initializer; -ByteBuffer::ByteBuffer(const Slice* slices, size_t nslices) { - // The following assertions check that the representation of a grpc::Slice is - // identical to that of a grpc_slice: it has a grpc_slice field, and nothing - // else. - static_assert(std::is_same<decltype(slices[0].slice_), grpc_slice>::value, - "Slice must have same representation as grpc_slice"); - static_assert(sizeof(Slice) == sizeof(grpc_slice), - "Slice must have same representation as grpc_slice"); - // The following assertions check that the representation of a ByteBuffer is - // identical to grpc_byte_buffer*: it has a grpc_byte_buffer* field, - // and nothing else. - static_assert(std::is_same<decltype(buffer_), grpc_byte_buffer*>::value, - "ByteBuffer must have same representation as " - "grpc_byte_buffer*"); - static_assert(sizeof(ByteBuffer) == sizeof(grpc_byte_buffer*), - "ByteBuffer must have same representation as " - "grpc_byte_buffer*"); - g_gli_initializer.summon(); // Make sure that initializer linked in - // The const_cast is legal if grpc_raw_byte_buffer_create() does no more - // than its advertised side effect of increasing the reference count of the - // slices it processes, and such an increase does not affect the semantics - // seen by the caller of this constructor. - buffer_ = grpc_raw_byte_buffer_create( - reinterpret_cast<grpc_slice*>(const_cast<Slice*>(slices)), nslices); -} - Status ByteBuffer::Dump(std::vector<Slice>* slices) const { slices->clear(); if (!buffer_) { @@ -69,14 +43,6 @@ Status ByteBuffer::Dump(std::vector<Slice>* slices) const { return Status::OK; } -size_t ByteBuffer::Length() const { - if (buffer_) { - return grpc_byte_buffer_length(buffer_); - } else { - return 0; - } -} - ByteBuffer::ByteBuffer(const ByteBuffer& buf) : buffer_(grpc_byte_buffer_copy(buf.buffer_)) {} @@ -90,10 +56,4 @@ ByteBuffer& ByteBuffer::operator=(const ByteBuffer& buf) { return *this; } -void ByteBuffer::Swap(ByteBuffer* other) { - grpc_byte_buffer* tmp = other->buffer_; - other->buffer_ = buffer_; - buffer_ = tmp; -} - } // namespace grpc diff --git a/src/cpp/util/slice_cc.cc b/src/cpp/util/slice_cc.cc deleted file mode 100644 index c72dbdbfc0..0000000000 --- a/src/cpp/util/slice_cc.cc +++ /dev/null @@ -1,55 +0,0 @@ -/* - * - * Copyright 2015 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#include <grpc/slice.h> -#include <grpcpp/support/slice.h> - -namespace grpc { - -Slice::Slice() : slice_(grpc_empty_slice()) {} - -Slice::~Slice() { grpc_slice_unref(slice_); } - -Slice::Slice(grpc_slice slice, AddRef) : slice_(grpc_slice_ref(slice)) {} - -Slice::Slice(grpc_slice slice, StealRef) : slice_(slice) {} - -Slice::Slice(size_t len) : slice_(grpc_slice_malloc(len)) {} - -Slice::Slice(const void* buf, size_t len) - : slice_( - grpc_slice_from_copied_buffer(static_cast<const char*>(buf), len)) {} - -Slice::Slice(const grpc::string& str) - : slice_(grpc_slice_from_copied_buffer(str.c_str(), str.length())) {} - -Slice::Slice(const void* buf, size_t len, StaticSlice) - : slice_( - grpc_slice_from_static_buffer(static_cast<const char*>(buf), len)) {} - -Slice::Slice(const Slice& other) : slice_(grpc_slice_ref(other.slice_)) {} - -Slice::Slice(void* buf, size_t len, void (*destroy)(void*), void* user_data) - : slice_(grpc_slice_new_with_user_data(buf, len, destroy, user_data)) {} - -Slice::Slice(void* buf, size_t len, void (*destroy)(void*, size_t)) - : slice_(grpc_slice_new_with_len(buf, len, destroy)) {} - -grpc_slice Slice::c_slice() const { return grpc_slice_ref(slice_); } - -} // namespace grpc diff --git a/src/csharp/Grpc.Core/Version.csproj.include b/src/csharp/Grpc.Core/Version.csproj.include index 9b55f2469a..f7a7a5cbe9 100755 --- a/src/csharp/Grpc.Core/Version.csproj.include +++ b/src/csharp/Grpc.Core/Version.csproj.include @@ -1,7 +1,7 @@ <!-- This file is generated --> <Project> <PropertyGroup> - <GrpcCsharpVersion>1.11.0-dev</GrpcCsharpVersion> + <GrpcCsharpVersion>1.12.0-dev</GrpcCsharpVersion> <GoogleProtobufVersion>3.3.0</GoogleProtobufVersion> </PropertyGroup> </Project> diff --git a/src/csharp/Grpc.Core/VersionInfo.cs b/src/csharp/Grpc.Core/VersionInfo.cs index 2902aee8d9..06a0396c34 100644 --- a/src/csharp/Grpc.Core/VersionInfo.cs +++ b/src/csharp/Grpc.Core/VersionInfo.cs @@ -33,11 +33,11 @@ namespace Grpc.Core /// <summary> /// Current <c>AssemblyFileVersion</c> of gRPC C# assemblies /// </summary> - public const string CurrentAssemblyFileVersion = "1.11.0.0"; + public const string CurrentAssemblyFileVersion = "1.12.0.0"; /// <summary> /// Current version of gRPC C# /// </summary> - public const string CurrentVersion = "1.11.0-dev"; + public const string CurrentVersion = "1.12.0-dev"; } } diff --git a/src/csharp/build_packages_dotnetcli.bat b/src/csharp/build_packages_dotnetcli.bat index 76391738ff..0d65748697 100755 --- a/src/csharp/build_packages_dotnetcli.bat +++ b/src/csharp/build_packages_dotnetcli.bat @@ -13,7 +13,7 @@ @rem limitations under the License. @rem Current package versions -set VERSION=1.11.0-dev +set VERSION=1.12.0-dev @rem Adjust the location of nuget.exe set NUGET=C:\nuget\nuget.exe diff --git a/src/csharp/build_packages_dotnetcli.sh b/src/csharp/build_packages_dotnetcli.sh index 1b73614b91..66aba36089 100755 --- a/src/csharp/build_packages_dotnetcli.sh +++ b/src/csharp/build_packages_dotnetcli.sh @@ -45,7 +45,7 @@ dotnet pack --configuration Release Grpc.Auth --output ../../../artifacts dotnet pack --configuration Release Grpc.HealthCheck --output ../../../artifacts dotnet pack --configuration Release Grpc.Reflection --output ../../../artifacts -nuget pack Grpc.nuspec -Version "1.11.0-dev" -OutputDirectory ../../artifacts -nuget pack Grpc.Tools.nuspec -Version "1.11.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.nuspec -Version "1.12.0-dev" -OutputDirectory ../../artifacts +nuget pack Grpc.Tools.nuspec -Version "1.12.0-dev" -OutputDirectory ../../artifacts (cd ../../artifacts && zip csharp_nugets_dotnetcli.zip *.nupkg) diff --git a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec index 954beed8e1..f06312bea9 100644 --- a/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec +++ b/src/objective-c/!ProtoCompiler-gRPCPlugin.podspec @@ -42,7 +42,7 @@ Pod::Spec.new do |s| # exclamation mark ensures that other "regular" pods will be able to find it as it'll be installed # before them. s.name = '!ProtoCompiler-gRPCPlugin' - v = '1.11.0-dev' + v = '1.12.0-dev' s.version = v s.summary = 'The gRPC ProtoC plugin generates Objective-C files from .proto services.' s.description = <<-DESC diff --git a/src/objective-c/GRPCClient/GRPCCall.m b/src/objective-c/GRPCClient/GRPCCall.m index cf70064298..501114dea0 100644 --- a/src/objective-c/GRPCClient/GRPCCall.m +++ b/src/objective-c/GRPCClient/GRPCCall.m @@ -355,17 +355,8 @@ static NSString * const kBearerPrefix = @"Bearer "; } dispatch_async(_callQueue, ^{ - __weak GRPCCall *weakSelf = self; - [self writeMessage:value withErrorHandler:^{ - __strong GRPCCall *strongSelf = weakSelf; - if (strongSelf != nil) { - [strongSelf maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain - code:GRPCErrorCodeInternal - userInfo:nil]]; - // Wrapped call must be canceled when error is reported to upper layers - [strongSelf cancelCall]; - } - }]; + // Write error is not processed here. It is handled by op batch of GRPC_OP_RECV_STATUS_ON_CLIENT + [self writeMessage:value withErrorHandler:nil]; }); } @@ -387,15 +378,8 @@ static NSString * const kBearerPrefix = @"Bearer "; [self cancel]; } else { dispatch_async(_callQueue, ^{ - __weak GRPCCall *weakSelf = self; - [self finishRequestWithErrorHandler:^{ - __strong GRPCCall *strongSelf = weakSelf; - [strongSelf maybeFinishWithError:[NSError errorWithDomain:kGRPCErrorDomain - code:GRPCErrorCodeInternal - userInfo:nil]]; - // Wrapped call must be canceled when error is reported to upper layers - [strongSelf cancelCall]; - }]; + // EOS error is not processed here. It is handled by op batch of GRPC_OP_RECV_STATUS_ON_CLIENT + [self finishRequestWithErrorHandler:nil]; }); } } diff --git a/src/objective-c/GRPCClient/private/version.h b/src/objective-c/GRPCClient/private/version.h index 405c2fff9f..a99eb289c3 100644 --- a/src/objective-c/GRPCClient/private/version.h +++ b/src/objective-c/GRPCClient/private/version.h @@ -23,4 +23,4 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.11.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.12.0-dev" diff --git a/src/objective-c/tests/GRPCClientTests.m b/src/objective-c/tests/GRPCClientTests.m index 3bab7f6671..e716198009 100644 --- a/src/objective-c/tests/GRPCClientTests.m +++ b/src/objective-c/tests/GRPCClientTests.m @@ -31,6 +31,8 @@ #import <RxLibrary/GRXWriter+Immediate.h> #import <RxLibrary/GRXBufferedPipe.h> +#include <netinet/in.h> + #import "version.h" #define TEST_TIMEOUT 16 @@ -482,4 +484,40 @@ static GRPCProtoMethod *kFullDuplexCallMethod; [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; } +- (int)findFreePort { + struct sockaddr_in addr; + unsigned int addr_len = sizeof(addr); + memset(&addr, 0, sizeof(addr)); + addr.sin_family = AF_INET; + int fd = socket(AF_INET, SOCK_STREAM, 0); + XCTAssertEqual(bind(fd, (struct sockaddr*)&addr, sizeof(addr)), 0); + XCTAssertEqual(getsockname(fd, (struct sockaddr*)&addr, &addr_len), 0); + XCTAssertEqual(addr_len, sizeof(addr)); + close(fd); + return addr.sin_port; +} + +- (void)testErrorCode { + int port = [self findFreePort]; + NSString * const kDummyAddress = [NSString stringWithFormat:@"localhost:%d", port]; + __weak XCTestExpectation *completion = [self expectationWithDescription:@"Empty RPC completed."]; + + GRPCCall *call = [[GRPCCall alloc] initWithHost:kDummyAddress + path:kEmptyCallMethod.HTTPPath + requestsWriter:[GRXWriter writerWithValue:[NSData data]]]; + + id<GRXWriteable> responsesWriteable = [[GRXWriteable alloc] initWithValueHandler:^(NSData *value) { + // Should not reach here + XCTAssert(NO); + } completionHandler:^(NSError *errorOrNil) { + XCTAssertNotNil(errorOrNil, @"Finished with no error"); + XCTAssertEqual(errorOrNil.code, GRPC_STATUS_UNAVAILABLE); + [completion fulfill]; + }]; + + [call startWithWriteable:responsesWriteable]; + + [self waitForExpectationsWithTimeout:TEST_TIMEOUT handler:nil]; +} + @end diff --git a/src/objective-c/tests/version.h b/src/objective-c/tests/version.h index 6f6cd25007..15fe1a59f3 100644 --- a/src/objective-c/tests/version.h +++ b/src/objective-c/tests/version.h @@ -23,5 +23,5 @@ // `tools/buildgen/generate_projects.sh`. -#define GRPC_OBJC_VERSION_STRING @"1.11.0-dev" +#define GRPC_OBJC_VERSION_STRING @"1.12.0-dev" #define GRPC_C_VERSION_STRING @"6.0.0-dev" diff --git a/src/php/composer.json b/src/php/composer.json index dbf0cc35fd..57d911db79 100644 --- a/src/php/composer.json +++ b/src/php/composer.json @@ -2,7 +2,7 @@ "name": "grpc/grpc-dev", "description": "gRPC library for PHP - for Developement use only", "license": "Apache-2.0", - "version": "1.11.0", + "version": "1.12.0", "require": { "php": ">=5.5.0", "google/protobuf": "^v3.3.0" diff --git a/src/php/ext/grpc/version.h b/src/php/ext/grpc/version.h index dd2a701ada..37df2768bf 100644 --- a/src/php/ext/grpc/version.h +++ b/src/php/ext/grpc/version.h @@ -20,6 +20,6 @@ #ifndef VERSION_H #define VERSION_H -#define PHP_GRPC_VERSION "1.11.0dev" +#define PHP_GRPC_VERSION "1.12.0dev" #endif /* VERSION_H */ diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc_gevent.pxd b/src/python/grpcio/grpc/_cython/_cygrpc/grpc_gevent.pxd index f35aebadde..f5688d08cd 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc_gevent.pxd +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc_gevent.pxd @@ -83,8 +83,6 @@ cdef extern from "src/core/lib/iomgr/tcp_custom.h": const grpc_sockaddr* addr, int* len); grpc_error* (*getsockname)(grpc_custom_socket* socket, const grpc_sockaddr* addr, int* len); - grpc_error* (*setsockopt)(grpc_custom_socket* socket, int level, int optname, - const void* optval, uint32_t len); grpc_error* (*bind)(grpc_custom_socket* socket, const grpc_sockaddr* addr, size_t len, int flags); grpc_error* (*listen)(grpc_custom_socket* socket); diff --git a/src/python/grpcio/grpc/_cython/_cygrpc/grpc_gevent.pyx b/src/python/grpcio/grpc/_cython/_cygrpc/grpc_gevent.pyx index ba8c731891..31ef671aed 100644 --- a/src/python/grpcio/grpc/_cython/_cygrpc/grpc_gevent.pyx +++ b/src/python/grpcio/grpc/_cython/_cygrpc/grpc_gevent.pyx @@ -239,11 +239,6 @@ cdef grpc_error* socket_getsockname(grpc_custom_socket* socket, length[0] = c_addr.len return grpc_error_none() -cdef grpc_error* socket_setsockopt(grpc_custom_socket* socket, int level, int optname, - const void *optval, uint32_t optlen) with gil: - # No-op; we provide a default set of options - return grpc_error_none() - def applysockopts(s): s.setsockopt(gevent_socket.SOL_SOCKET, gevent_socket.SO_REUSEADDR, 1) s.setsockopt(gevent_socket.IPPROTO_TCP, gevent_socket.TCP_NODELAY, True) @@ -435,7 +430,6 @@ def init_grpc_gevent(): gevent_socket_vtable.read = socket_read gevent_socket_vtable.getpeername = socket_getpeername gevent_socket_vtable.getsockname = socket_getsockname - gevent_socket_vtable.setsockopt = socket_setsockopt gevent_socket_vtable.bind = socket_bind gevent_socket_vtable.listen = socket_listen gevent_socket_vtable.accept = socket_accept diff --git a/src/python/grpcio/grpc/_grpcio_metadata.py b/src/python/grpcio/grpc/_grpcio_metadata.py index 4a69d859fc..cb5da72f1f 100644 --- a/src/python/grpcio/grpc/_grpcio_metadata.py +++ b/src/python/grpcio/grpc/_grpcio_metadata.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc/_grpcio_metadata.py.template`!!! -__version__ = """1.11.0.dev0""" +__version__ = """1.12.0.dev0""" diff --git a/src/python/grpcio/grpc_version.py b/src/python/grpcio/grpc_version.py index 32e82493f3..de5a780abd 100644 --- a/src/python/grpcio/grpc_version.py +++ b/src/python/grpcio/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio/grpc_version.py.template`!!! -VERSION = '1.11.0.dev0' +VERSION = '1.12.0.dev0' diff --git a/src/python/grpcio_health_checking/grpc_version.py b/src/python/grpcio_health_checking/grpc_version.py index ad4c85cc12..afcd316e5c 100644 --- a/src/python/grpcio_health_checking/grpc_version.py +++ b/src/python/grpcio_health_checking/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_health_checking/grpc_version.py.template`!!! -VERSION = '1.11.0.dev0' +VERSION = '1.12.0.dev0' diff --git a/src/python/grpcio_reflection/grpc_version.py b/src/python/grpcio_reflection/grpc_version.py index 6322d847b1..824b73201d 100644 --- a/src/python/grpcio_reflection/grpc_version.py +++ b/src/python/grpcio_reflection/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_reflection/grpc_version.py.template`!!! -VERSION = '1.11.0.dev0' +VERSION = '1.12.0.dev0' diff --git a/src/python/grpcio_testing/grpc_version.py b/src/python/grpcio_testing/grpc_version.py index 1e75fea12e..5b1f4c4cc0 100644 --- a/src/python/grpcio_testing/grpc_version.py +++ b/src/python/grpcio_testing/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_testing/grpc_version.py.template`!!! -VERSION = '1.11.0.dev0' +VERSION = '1.12.0.dev0' diff --git a/src/python/grpcio_tests/grpc_version.py b/src/python/grpcio_tests/grpc_version.py index 0cd7bd257f..382f95018e 100644 --- a/src/python/grpcio_tests/grpc_version.py +++ b/src/python/grpcio_tests/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/src/python/grpcio_tests/grpc_version.py.template`!!! -VERSION = '1.11.0.dev0' +VERSION = '1.12.0.dev0' diff --git a/src/ruby/end2end/multiple_killed_watching_threads_driver.rb b/src/ruby/end2end/multiple_killed_watching_threads_driver.rb index 8f078cfbed..8ec2073d98 100755 --- a/src/ruby/end2end/multiple_killed_watching_threads_driver.rb +++ b/src/ruby/end2end/multiple_killed_watching_threads_driver.rb @@ -54,8 +54,14 @@ def run_multiple_killed_watches(num_threads, sleep_time) end def main + STDERR.puts '10 iterations, sleep 0.1 before killing thread' run_multiple_killed_watches(10, 0.1) + STDERR.puts '1000 iterations, sleep 0.001 before killing thread' run_multiple_killed_watches(1000, 0.001) + STDERR.puts '10000 iterations, sleep 0.00001 before killing thread' + run_multiple_killed_watches(10_000, 0.00001) + STDERR.puts '20000 iterations, sleep 0.00001 before killing thread' + run_multiple_killed_watches(20_000, 0.00001) end main diff --git a/src/ruby/ext/grpc/extconf.rb b/src/ruby/ext/grpc/extconf.rb index e8e87e4f15..4760f33e38 100644 --- a/src/ruby/ext/grpc/extconf.rb +++ b/src/ruby/ext/grpc/extconf.rb @@ -84,7 +84,7 @@ if grpc_config == 'gcov' end if grpc_config == 'dbg' - $CFLAGS << ' -O0' + $CFLAGS << ' -O0 -ggdb3' end $LDFLAGS << ' -Wl,-wrap,memcpy' if RUBY_PLATFORM =~ /linux/ diff --git a/src/ruby/ext/grpc/rb_channel.c b/src/ruby/ext/grpc/rb_channel.c index e8bfeb32a0..3f0dc530cf 100644 --- a/src/ruby/ext/grpc/rb_channel.c +++ b/src/ruby/ext/grpc/rb_channel.c @@ -315,7 +315,7 @@ static VALUE grpc_rb_channel_get_connectivity_state(int argc, VALUE* argv, } typedef struct watch_state_stack { - grpc_channel* channel; + bg_watched_channel* bg_wrapped; gpr_timespec deadline; int last_state; } watch_state_stack; @@ -328,15 +328,15 @@ static void* wait_for_watch_state_op_complete_without_gvl(void* arg) { gpr_mu_lock(&global_connection_polling_mu); // its unsafe to do a "watch" after "channel polling abort" because the cq has // been shut down. - if (abort_channel_polling) { + if (abort_channel_polling || stack->bg_wrapped->channel_destroyed) { gpr_mu_unlock(&global_connection_polling_mu); return (void*)0; } op = gpr_zalloc(sizeof(watch_state_op)); op->op_type = WATCH_STATE_API; - grpc_channel_watch_connectivity_state(stack->channel, stack->last_state, - stack->deadline, channel_polling_cq, - op); + grpc_channel_watch_connectivity_state(stack->bg_wrapped->channel, + stack->last_state, stack->deadline, + channel_polling_cq, op); while (!op->op.api_callback_args.called_back) { gpr_cv_wait(&global_connection_polling_cv, &global_connection_polling_mu, @@ -388,7 +388,7 @@ static VALUE grpc_rb_channel_watch_connectivity_state(VALUE self, return Qnil; } - stack.channel = wrapper->bg_wrapped->channel; + stack.bg_wrapped = wrapper->bg_wrapped; stack.deadline = grpc_rb_time_timeval(deadline, 0), stack.last_state = NUM2LONG(last_state); diff --git a/src/ruby/lib/grpc/generic/bidi_call.rb b/src/ruby/lib/grpc/generic/bidi_call.rb index 086455db0b..ffb232b827 100644 --- a/src/ruby/lib/grpc/generic/bidi_call.rb +++ b/src/ruby/lib/grpc/generic/bidi_call.rb @@ -124,12 +124,18 @@ module GRPC def read_using_run_batch ops = { RECV_MESSAGE => nil } ops[RECV_INITIAL_METADATA] = nil unless @metadata_received - batch_result = @call.run_batch(ops) - unless @metadata_received - @call.metadata = batch_result.metadata - @metadata_received = true + begin + batch_result = @call.run_batch(ops) + unless @metadata_received + @call.metadata = batch_result.metadata + @metadata_received = true + end + batch_result + rescue GRPC::Core::CallError => e + GRPC.logger.warn('bidi call: read_using_run_batch failed') + GRPC.logger.warn(e) + nil end - batch_result end # set_output_stream_done is relevant on client-side @@ -155,7 +161,12 @@ module GRPC GRPC.logger.debug("bidi-write-loop: #{count} writes done") if is_client GRPC.logger.debug("bidi-write-loop: client sent #{count}, waiting") - @call.run_batch(SEND_CLOSE_FROM_CLIENT => nil) + begin + @call.run_batch(SEND_CLOSE_FROM_CLIENT => nil) + rescue GRPC::Core::CallError => e + GRPC.logger.warn('bidi-write-loop: send close failed') + GRPC.logger.warn(e) + end GRPC.logger.debug('bidi-write-loop: done') end GRPC.logger.debug('bidi-write-loop: finished') @@ -187,7 +198,7 @@ module GRPC batch_result = read_using_run_batch # handle the next message - if batch_result.message.nil? + if batch_result.nil? || batch_result.message.nil? GRPC.logger.debug("bidi-read-loop: null batch #{batch_result}") if is_client diff --git a/src/ruby/lib/grpc/version.rb b/src/ruby/lib/grpc/version.rb index 256a543a9f..2cb7c4be53 100644 --- a/src/ruby/lib/grpc/version.rb +++ b/src/ruby/lib/grpc/version.rb @@ -14,5 +14,5 @@ # GRPC contains the General RPC module. module GRPC - VERSION = '1.11.0.dev' + VERSION = '1.12.0.dev' end diff --git a/src/ruby/spec/generic/client_stub_spec.rb b/src/ruby/spec/generic/client_stub_spec.rb index d858c4e3fe..da50f8d0c9 100644 --- a/src/ruby/spec/generic/client_stub_spec.rb +++ b/src/ruby/spec/generic/client_stub_spec.rb @@ -750,6 +750,90 @@ describe 'ClientStub' do # rubocop:disable Metrics/BlockLength expected_error_message) end end + + # Prompted by grpc/github #14853 + describe 'client-side error handling on bidi streams' do + class EnumeratorQueue + def initialize(queue) + @queue = queue + end + + def each + loop do + msg = @queue.pop + break if msg.nil? + yield msg + end + end + end + + def run_server_bidi_shutdown_after_one_read + @server.start + recvd_rpc = @server.request_call + recvd_call = recvd_rpc.call + server_call = GRPC::ActiveCall.new( + recvd_call, noop, noop, INFINITE_FUTURE, + metadata_received: true, started: false) + expect(server_call.remote_read).to eq('first message') + @server.shutdown_and_notify(from_relative_time(0)) + @server.close + end + + it 'receives a grpc status code when writes to a bidi stream fail' do + # This test tries to trigger the case when a 'SEND_MESSAGE' op + # and subseqeunt 'SEND_CLOSE_FROM_CLIENT' op of a bidi stream fails. + # In this case, iteration through the response stream should result + # in a grpc status code, and the writer thread should not raise an + # exception. + server_thread = Thread.new do + run_server_bidi_shutdown_after_one_read + end + stub = GRPC::ClientStub.new(@host, :this_channel_is_insecure) + request_queue = Queue.new + @sent_msgs = EnumeratorQueue.new(request_queue) + responses = get_responses(stub) + request_queue.push('first message') + # Now wait for the server to shut down. + server_thread.join + # Sanity check. This test is not interesting if + # Thread.abort_on_exception is not set. + expect(Thread.abort_on_exception).to be(true) + # An attempt to send a second message should fail now that the + # server is down. + request_queue.push('second message') + request_queue.push(nil) + expect { responses.next }.to raise_error(GRPC::BadStatus) + end + + def run_server_bidi_shutdown_after_one_write + @server.start + recvd_rpc = @server.request_call + recvd_call = recvd_rpc.call + server_call = GRPC::ActiveCall.new( + recvd_call, noop, noop, INFINITE_FUTURE, + metadata_received: true, started: false) + server_call.send_initial_metadata + server_call.remote_send('message') + @server.shutdown_and_notify(from_relative_time(0)) + @server.close + end + + it 'receives a grpc status code when reading from a failed bidi call' do + server_thread = Thread.new do + run_server_bidi_shutdown_after_one_write + end + stub = GRPC::ClientStub.new(@host, :this_channel_is_insecure) + request_queue = Queue.new + @sent_msgs = EnumeratorQueue.new(request_queue) + responses = get_responses(stub) + expect(responses.next).to eq('message') + # Wait for the server to shut down + server_thread.join + expect { responses.next }.to raise_error(GRPC::BadStatus) + # Push a sentinel to allow the writer thread to finish + request_queue.push(nil) + end + end end describe 'without a call operation' do @@ -810,6 +894,55 @@ describe 'ClientStub' do # rubocop:disable Metrics/BlockLength responses.each { |r| p r } end end + + def run_server_bidi_expect_client_to_cancel(wait_for_shutdown_ok_callback) + @server.start + recvd_rpc = @server.request_call + recvd_call = recvd_rpc.call + server_call = GRPC::ActiveCall.new( + recvd_call, noop, noop, INFINITE_FUTURE, + metadata_received: true, started: false) + server_call.send_initial_metadata + server_call.remote_send('server call received') + wait_for_shutdown_ok_callback.call + # since the client is cancelling the call, + # we should be able to shut down cleanly + @server.shutdown_and_notify(nil) + @server.close + end + + it 'receives a grpc status code when reading from a cancelled bidi call' do + # This test tries to trigger a 'RECV_INITIAL_METADATA' and/or + # 'RECV_MESSAGE' op failure. + # An attempt to read a message might fail; in that case, iteration + # through the response stream should still result in a grpc status. + server_can_shutdown = false + server_can_shutdown_mu = Mutex.new + server_can_shutdown_cv = ConditionVariable.new + wait_for_shutdown_ok_callback = proc do + server_can_shutdown_mu.synchronize do + server_can_shutdown_cv.wait(server_can_shutdown_mu) until server_can_shutdown + end + end + server_thread = Thread.new do + run_server_bidi_expect_client_to_cancel(wait_for_shutdown_ok_callback) + end + stub = GRPC::ClientStub.new(@host, :this_channel_is_insecure) + request_queue = Queue.new + @sent_msgs = EnumeratorQueue.new(request_queue) + responses = get_responses(stub) + expect(responses.next).to eq('server call received') + @op.cancel + expect { responses.next }.to raise_error(GRPC::Cancelled) + # Now let the server proceed to shut down. + server_can_shutdown_mu.synchronize do + server_can_shutdown = true + server_can_shutdown_cv.broadcast + end + server_thread.join + # Push a sentinel to allow the writer thread to finish + request_queue.push(nil) + end end end diff --git a/src/ruby/tools/version.rb b/src/ruby/tools/version.rb index 8dc1623d6f..822f70eb0a 100644 --- a/src/ruby/tools/version.rb +++ b/src/ruby/tools/version.rb @@ -14,6 +14,6 @@ module GRPC module Tools - VERSION = '1.11.0.dev' + VERSION = '1.12.0.dev' end end diff --git a/templates/config.w32.template b/templates/config.w32.template index 4edef963f2..ef25e55596 100644 --- a/templates/config.w32.template +++ b/templates/config.w32.template @@ -23,11 +23,13 @@ EXTENSION("grpc", grpc_source, null, "/DOPENSSL_NO_ASM /D_GNU_SOURCE /DWIN32_LEAN_AND_MEAN "+ "/D_HAS_EXCEPTIONS=0 /DNOMINMAX /DGRPC_ARES=0 /D_WIN32_WINNT=0x600 "+ + "/DPB_FIELD_16BIT "+ "/I"+configure_module_dirname+" "+ "/I"+configure_module_dirname+"\\include "+ "/I"+configure_module_dirname+"\\src\\php\\ext\\grpc "+ "/I"+configure_module_dirname+"\\third_party\\boringssl\\include "+ - "/I"+configure_module_dirname+"\\third_party\\zlib"); + "/I"+configure_module_dirname+"\\third_party\\zlib "+ + "/I"+configure_module_dirname+"\\third_party\\address_sorting\\include"); <% dirs = {} for lib in libs: diff --git a/templates/test/cpp/naming/resolver_component_tests_defs.include b/templates/test/cpp/naming/resolver_component_tests_defs.include index 01e1860545..c29b466125 100644 --- a/templates/test/cpp/naming/resolver_component_tests_defs.include +++ b/templates/test/cpp/naming/resolver_component_tests_defs.include @@ -36,7 +36,8 @@ if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then fi export GRPC_DNS_RESOLVER=ares -"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_dns_server_port" > /dev/null 2>&1 & +DNS_SERVER_LOG="$(mktemp)" +"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_dns_server_port" > "$DNS_SERVER_LOG" 2>&1 & DNS_SERVER_PID=$! echo "Local DNS server started. PID: $DNS_SERVER_PID" @@ -55,8 +56,11 @@ done if [[ $RETRY == 1 ]]; then echo "FAILED TO START LOCAL DNS SERVER" - kill -SIGTERM "$DNS_SERVER_PID" + kill -SIGTERM "$DNS_SERVER_PID" || true wait + echo "========== DNS server log (merged stdout and stderr) =========" + cat "$DNS_SERVER_LOG" + echo "========== end DNS server log ================================" exit 1 fi diff --git a/templates/tools/dockerfile/interoptest/grpc_interop_android_java/Dockerfile.template b/templates/tools/dockerfile/interoptest/grpc_interop_android_java/Dockerfile.template deleted file mode 100644 index 1f6755eb51..0000000000 --- a/templates/tools/dockerfile/interoptest/grpc_interop_android_java/Dockerfile.template +++ /dev/null @@ -1,80 +0,0 @@ -%YAML 1.2 ---- | - # Copyright 2017 gRPC authors. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - FROM debian:jessie - - # Install JDK 8 and Git - RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && ${'\\'} - echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list && ${'\\'} - echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list && ${'\\'} - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 - RUN apt-get update && apt-get -y install ${'\\'} - git ${'\\'} - libapr1 ${'\\'} - oracle-java8-installer ${'\\'} - && ${'\\'} - apt-get clean && rm -r /var/cache/oracle-jdk8-installer/ - ENV JAVA_HOME /usr/lib/jvm/java-8-oracle - ENV PATH $PATH:$JAVA_HOME/bin - - # Install protobuf - RUN apt-get update && apt-get install -y ${'\\'} - autoconf ${'\\'} - build-essential ${'\\'} - curl ${'\\'} - gcc ${'\\'} - libtool ${'\\'} - unzip ${'\\'} - && ${'\\'} - apt-get clean - WORKDIR / - RUN git clone https://github.com/google/protobuf.git - WORKDIR /protobuf - RUN git checkout v3.3.1 && ${'\\'} - ./autogen.sh && ${'\\'} - ./configure && ${'\\'} - make && ${'\\'} - make check && ${'\\'} - make install - - # Install gcloud command line tools - ENV CLOUD_SDK_REPO "cloud-sdk-jessie" - RUN echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && ${'\\'} - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && ${'\\'} - apt-get update && apt-get install -y google-cloud-sdk && apt-get clean && ${'\\'} - gcloud config set component_manager/disable_update_check true - - # Install Android SDK - WORKDIR / - RUN mkdir android-sdk - WORKDIR android-sdk - RUN wget -q https://dl.google.com/android/repository/tools_r25.2.5-linux.zip && ${'\\'} - unzip -qq tools_r25.2.5-linux.zip && ${'\\'} - rm tools_r25.2.5-linux.zip && ${'\\'} - echo y | tools/bin/sdkmanager "platforms;android-22" && ${'\\'} - echo y | tools/bin/sdkmanager "build-tools;25.0.2" && ${'\\'} - echo y | tools/bin/sdkmanager "extras;android;m2repository" && ${'\\'} - echo y | tools/bin/sdkmanager "extras;google;google_play_services" && ${'\\'} - echo y | tools/bin/sdkmanager "extras;google;m2repository" && ${'\\'} - echo y | tools/bin/sdkmanager "patcher;v4" && ${'\\'} - echo y | tools/bin/sdkmanager "platform-tools" - ENV ANDROID_HOME "/android-sdk" - - # Reset the working directory - WORKDIR / - - # Define the default command. - CMD ["bash"] diff --git a/templates/tools/dockerfile/python_deps.include b/templates/tools/dockerfile/python_deps.include index cd1af22b43..ce36de8ad9 100644 --- a/templates/tools/dockerfile/python_deps.include +++ b/templates/tools/dockerfile/python_deps.include @@ -9,6 +9,6 @@ RUN apt-get update && apt-get install -y ${'\\'} python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/templates/tools/dockerfile/test/cxx_alpine_x64/Dockerfile.template b/templates/tools/dockerfile/test/cxx_alpine_x64/Dockerfile.template index 1bbfdfdf25..b250566861 100644 --- a/templates/tools/dockerfile/test/cxx_alpine_x64/Dockerfile.template +++ b/templates/tools/dockerfile/test/cxx_alpine_x64/Dockerfile.template @@ -34,12 +34,13 @@ strace ${'\\'} python-dev ${'\\'} py-pip ${'\\'} + py-yaml ${'\\'} unzip ${'\\'} wget ${'\\'} zip # Install Python packages from PyPI - RUN pip install --upgrade pip==9.0.1 + RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/test/core/surface/server_chttp2_test.cc b/test/core/surface/server_chttp2_test.cc index f0412d01d9..fd8ab9cd3d 100644 --- a/test/core/surface/server_chttp2_test.cc +++ b/test/core/surface/server_chttp2_test.cc @@ -37,6 +37,8 @@ void test_unparsable_target(void) { grpc_server_destroy(server); } +// GRPC_ARG_ALLOW_REUSEPORT isn't supported for custom servers +#ifndef GRPC_UV void test_add_same_port_twice() { grpc_arg a; a.type = GRPC_ARG_INTEGER; @@ -62,12 +64,15 @@ void test_add_same_port_twice() { grpc_server_destroy(server); grpc_completion_queue_destroy(cq); } +#endif int main(int argc, char** argv) { grpc_test_init(argc, argv); grpc_init(); test_unparsable_target(); +#ifndef GRPC_UV test_add_same_port_twice(); +#endif grpc_shutdown(); return 0; } diff --git a/test/core/surface/server_test.cc b/test/core/surface/server_test.cc index 3b08efb563..b4eabd8d4d 100644 --- a/test/core/surface/server_test.cc +++ b/test/core/surface/server_test.cc @@ -69,6 +69,8 @@ void test_request_call_on_no_server_cq(void) { grpc_server_destroy(server); } +// GRPC_ARG_ALLOW_REUSEPORT isn't supported for custom servers +#ifndef GRPC_UV void test_bind_server_twice(void) { grpc_arg a; a.type = GRPC_ARG_INTEGER; @@ -100,6 +102,7 @@ void test_bind_server_twice(void) { grpc_completion_queue_destroy(cq); gpr_free(addr); } +#endif void test_bind_server_to_addr(const char* host, bool secure) { int port = grpc_pick_unused_port_or_die(); @@ -149,7 +152,9 @@ int main(int argc, char** argv) { grpc_init(); test_register_method_fail(); test_request_call_on_no_server_cq(); +#ifndef GRPC_UV test_bind_server_twice(); +#endif static const char* addrs[] = { "::1", "127.0.0.1", "::ffff:127.0.0.1", "localhost", "0.0.0.0", "::", diff --git a/test/cpp/codegen/proto_utils_test.cc b/test/cpp/codegen/proto_utils_test.cc index 836d3d8076..53e2a2864c 100644 --- a/test/cpp/codegen/proto_utils_test.cc +++ b/test/cpp/codegen/proto_utils_test.cc @@ -24,32 +24,43 @@ #include <gtest/gtest.h> namespace grpc { + namespace internal { -// Provide access to GrpcBufferWriter internals. -class GrpcBufferWriterPeer { +// Provide access to GrpcProtoBufferWriter internals. +class GrpcProtoBufferWriterPeer { public: - explicit GrpcBufferWriterPeer(internal::GrpcBufferWriter* writer) + explicit GrpcProtoBufferWriterPeer(GrpcProtoBufferWriter* writer) : writer_(writer) {} bool have_backup() const { return writer_->have_backup_; } const grpc_slice& backup_slice() const { return writer_->backup_slice_; } const grpc_slice& slice() const { return writer_->slice_; } private: - GrpcBufferWriter* writer_; + GrpcProtoBufferWriter* writer_; +}; + +// Provide access to ByteBuffer internals. +class GrpcByteBufferPeer { + public: + explicit GrpcByteBufferPeer(ByteBuffer* bb) : bb_(bb) {} + grpc_byte_buffer* c_buffer() { return bb_->c_buffer(); } + + private: + ByteBuffer* bb_; }; class ProtoUtilsTest : public ::testing::Test {}; // Regression test for a memory corruption bug where a series of -// GrpcBufferWriter Next()/Backup() invocations could result in a dangling +// GrpcProtoBufferWriter Next()/Backup() invocations could result in a dangling // pointer returned by Next() due to the interaction between grpc_slice inlining // and GRPC_SLICE_START_PTR. TEST_F(ProtoUtilsTest, TinyBackupThenNext) { - grpc_byte_buffer* bp; + ByteBuffer bp; const int block_size = 1024; - GrpcBufferWriter writer(&bp, block_size, 8192); - GrpcBufferWriterPeer peer(&writer); + GrpcProtoBufferWriter writer(&bp, block_size, 8192); + GrpcProtoBufferWriterPeer peer(&writer); void* data; int size; @@ -63,17 +74,14 @@ TEST_F(ProtoUtilsTest, TinyBackupThenNext) { ASSERT_TRUE(writer.Next(&data, &size)); EXPECT_TRUE(peer.slice().refcount != nullptr); EXPECT_EQ(block_size, size); - - // Cleanup. - g_core_codegen_interface->grpc_byte_buffer_destroy(bp); } namespace { // Set backup_size to 0 to indicate no backup is needed. void BufferWriterTest(int block_size, int total_size, int backup_size) { - grpc_byte_buffer* bp; - GrpcBufferWriter writer(&bp, block_size, total_size); + ByteBuffer bb; + GrpcProtoBufferWriter writer(&bb, block_size, total_size); int written_size = 0; void* data; @@ -110,10 +118,11 @@ void BufferWriterTest(int block_size, int total_size, int backup_size) { writer.BackUp(backup_size); } } - EXPECT_EQ(grpc_byte_buffer_length(bp), (size_t)total_size); + EXPECT_EQ(bb.Length(), (size_t)total_size); grpc_byte_buffer_reader reader; - grpc_byte_buffer_reader_init(&reader, bp); + GrpcByteBufferPeer peer(&bb); + grpc_byte_buffer_reader_init(&reader, peer.c_buffer()); int read_bytes = 0; while (read_bytes < total_size) { grpc_slice s; @@ -126,7 +135,6 @@ void BufferWriterTest(int block_size, int total_size, int backup_size) { } EXPECT_EQ(read_bytes, total_size); grpc_byte_buffer_reader_destroy(&reader); - grpc_byte_buffer_destroy(bp); } TEST(WriterTest, TinyBlockTinyBackup) { @@ -154,7 +162,7 @@ TEST(WriterTest, LargeBlockLargeBackup) { BufferWriterTest(4096, 8192, 4095); } } // namespace grpc int main(int argc, char** argv) { - // Ensure the GrpcBufferWriter internals are initialized. + // Ensure the GrpcProtoBufferWriter internals are initialized. grpc::internal::GrpcLibraryInitializer init; init.summon(); grpc::GrpcLibraryCodegen lib; diff --git a/test/cpp/naming/resolver_component_tests_runner.sh b/test/cpp/naming/resolver_component_tests_runner.sh index 18b56ab69b..3f8765fa94 100755 --- a/test/cpp/naming/resolver_component_tests_runner.sh +++ b/test/cpp/naming/resolver_component_tests_runner.sh @@ -36,7 +36,8 @@ if [[ "$GRPC_DNS_RESOLVER" != "" && "$GRPC_DNS_RESOLVER" != ares ]]; then fi export GRPC_DNS_RESOLVER=ares -"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_dns_server_port" > /dev/null 2>&1 & +DNS_SERVER_LOG="$(mktemp)" +"$FLAGS_dns_server_bin_path" --records_config_path="$FLAGS_records_config_path" --port="$FLAGS_dns_server_port" > "$DNS_SERVER_LOG" 2>&1 & DNS_SERVER_PID=$! echo "Local DNS server started. PID: $DNS_SERVER_PID" @@ -55,8 +56,11 @@ done if [[ $RETRY == 1 ]]; then echo "FAILED TO START LOCAL DNS SERVER" - kill -SIGTERM "$DNS_SERVER_PID" + kill -SIGTERM "$DNS_SERVER_PID" || true wait + echo "========== DNS server log (merged stdout and stderr) =========" + cat "$DNS_SERVER_LOG" + echo "========== end DNS server log ================================" exit 1 fi diff --git a/test/cpp/server/server_builder_test.cc b/test/cpp/server/server_builder_test.cc index 2752b5838c..5c22dda16f 100644 --- a/test/cpp/server/server_builder_test.cc +++ b/test/cpp/server/server_builder_test.cc @@ -39,7 +39,10 @@ grpc::string MakePort() { return s.str(); } -grpc::string g_port = MakePort(); +const grpc::string& GetPort() { + static grpc::string g_port = MakePort(); + return g_port; +} TEST(ServerBuilderTest, NoOp) { ServerBuilder b; } @@ -50,7 +53,7 @@ TEST(ServerBuilderTest, CreateServerNoPorts) { TEST(ServerBuilderTest, CreateServerOnePort) { ServerBuilder() .RegisterService(&g_service) - .AddListeningPort(g_port, InsecureServerCredentials()) + .AddListeningPort(GetPort(), InsecureServerCredentials()) .BuildAndStart() ->Shutdown(); } @@ -58,8 +61,8 @@ TEST(ServerBuilderTest, CreateServerOnePort) { TEST(ServerBuilderTest, CreateServerRepeatedPort) { ServerBuilder() .RegisterService(&g_service) - .AddListeningPort(g_port, InsecureServerCredentials()) - .AddListeningPort(g_port, InsecureServerCredentials()) + .AddListeningPort(GetPort(), InsecureServerCredentials()) + .AddListeningPort(GetPort(), InsecureServerCredentials()) .BuildAndStart() ->Shutdown(); } @@ -67,8 +70,8 @@ TEST(ServerBuilderTest, CreateServerRepeatedPort) { TEST(ServerBuilderTest, CreateServerRepeatedPortWithDisallowedReusePort) { EXPECT_EQ(ServerBuilder() .RegisterService(&g_service) - .AddListeningPort(g_port, InsecureServerCredentials()) - .AddListeningPort(g_port, InsecureServerCredentials()) + .AddListeningPort(GetPort(), InsecureServerCredentials()) + .AddListeningPort(GetPort(), InsecureServerCredentials()) .AddChannelArgument(GRPC_ARG_ALLOW_REUSEPORT, 0) .BuildAndStart(), nullptr); diff --git a/test/cpp/util/byte_buffer_test.cc b/test/cpp/util/byte_buffer_test.cc index 605ef15123..47a5b7f03a 100644 --- a/test/cpp/util/byte_buffer_test.cc +++ b/test/cpp/util/byte_buffer_test.cc @@ -16,7 +16,8 @@ * */ -#include <grpcpp/support/byte_buffer.h> +#include <grpc++/support/byte_buffer.h> +#include <grpcpp/impl/grpc_library.h> #include <cstring> #include <vector> @@ -27,6 +28,9 @@ #include <gtest/gtest.h> namespace grpc { + +static internal::GrpcLibraryInitializer g_gli_initializer; + namespace { const char* kContent1 = "hello xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"; diff --git a/test/cpp/util/slice_test.cc b/test/cpp/util/slice_test.cc index 8e06062360..dc1910038f 100644 --- a/test/cpp/util/slice_test.cc +++ b/test/cpp/util/slice_test.cc @@ -16,13 +16,17 @@ * */ -#include <grpcpp/support/slice.h> +#include <grpc++/support/slice.h> +#include <grpcpp/impl/grpc_library.h> #include <grpc/grpc.h> #include <grpc/slice.h> #include <gtest/gtest.h> namespace grpc { + +static internal::GrpcLibraryInitializer g_gli_initializer; + namespace { const char* kContent = "hello xxxxxxxxxxxxxxxxxxxx world"; diff --git a/tools/distrib/pylint_code.sh b/tools/distrib/pylint_code.sh index 7175f1e15b..bbacd48737 100755 --- a/tools/distrib/pylint_code.sh +++ b/tools/distrib/pylint_code.sh @@ -29,7 +29,7 @@ VIRTUALENV=python_pylint_venv virtualenv $VIRTUALENV PYTHON=$(realpath $VIRTUALENV/bin/python) -$PYTHON -m pip install --upgrade pip==9.0.1 +$PYTHON -m pip install --upgrade pip==9.0.2 $PYTHON -m pip install pylint==1.6.5 for dir in "${DIRS[@]}"; do diff --git a/tools/distrib/python/grpcio_tools/grpc_version.py b/tools/distrib/python/grpcio_tools/grpc_version.py index e8ca6851eb..e71f836f6d 100644 --- a/tools/distrib/python/grpcio_tools/grpc_version.py +++ b/tools/distrib/python/grpcio_tools/grpc_version.py @@ -14,4 +14,4 @@ # AUTO-GENERATED FROM `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/grpc_version.py.template`!!! -VERSION = '1.11.0.dev0' +VERSION = '1.12.0.dev0' diff --git a/tools/distrib/yapf_code.sh b/tools/distrib/yapf_code.sh index d188a0242c..919e9c13d5 100755 --- a/tools/distrib/yapf_code.sh +++ b/tools/distrib/yapf_code.sh @@ -32,7 +32,7 @@ VIRTUALENV=yapf_virtual_environment virtualenv $VIRTUALENV PYTHON=$(realpath "${VIRTUALENV}/bin/python") -$PYTHON -m pip install --upgrade pip==9.0.1 +$PYTHON -m pip install --upgrade pip==9.0.2 $PYTHON -m pip install --upgrade futures $PYTHON -m pip install yapf==0.20.0 diff --git a/tools/dockerfile/grpc_clang_tidy/Dockerfile b/tools/dockerfile/grpc_clang_tidy/Dockerfile index eeba455341..d9599af45b 100644 --- a/tools/dockerfile/grpc_clang_tidy/Dockerfile +++ b/tools/dockerfile/grpc_clang_tidy/Dockerfile @@ -33,7 +33,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_android_java/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_android_java/Dockerfile deleted file mode 100644 index 519cdbfafa..0000000000 --- a/tools/dockerfile/interoptest/grpc_interop_android_java/Dockerfile +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM debian:jessie - -# Install JDK 8 and Git -RUN echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections && \ - echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee /etc/apt/sources.list.d/webupd8team-java.list && \ - echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | tee -a /etc/apt/sources.list.d/webupd8team-java.list && \ - apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886 -RUN apt-get update && apt-get -y install \ - git \ - libapr1 \ - oracle-java8-installer \ - && \ - apt-get clean && rm -r /var/cache/oracle-jdk8-installer/ -ENV JAVA_HOME /usr/lib/jvm/java-8-oracle -ENV PATH $PATH:$JAVA_HOME/bin - -# Install protobuf -RUN apt-get update && apt-get install -y \ - autoconf \ - build-essential \ - curl \ - gcc \ - libtool \ - unzip \ - && \ - apt-get clean -WORKDIR / -RUN git clone https://github.com/google/protobuf.git -WORKDIR /protobuf -RUN git checkout v3.3.1 && \ - ./autogen.sh && \ - ./configure && \ - make && \ - make check && \ - make install - -# Install gcloud command line tools -ENV CLOUD_SDK_REPO "cloud-sdk-jessie" -RUN echo "deb http://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \ - curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - && \ - apt-get update && apt-get install -y google-cloud-sdk && apt-get clean && \ - gcloud config set component_manager/disable_update_check true - -# Install Android SDK -WORKDIR / -RUN mkdir android-sdk -WORKDIR android-sdk -RUN wget -q https://dl.google.com/android/repository/tools_r25.2.5-linux.zip && \ - unzip -qq tools_r25.2.5-linux.zip && \ - rm tools_r25.2.5-linux.zip && \ - echo y | tools/bin/sdkmanager "platforms;android-22" && \ - echo y | tools/bin/sdkmanager "build-tools;25.0.2" && \ - echo y | tools/bin/sdkmanager "extras;android;m2repository" && \ - echo y | tools/bin/sdkmanager "extras;google;google_play_services" && \ - echo y | tools/bin/sdkmanager "extras;google;m2repository" && \ - echo y | tools/bin/sdkmanager "patcher;v4" && \ - echo y | tools/bin/sdkmanager "platform-tools" -ENV ANDROID_HOME "/android-sdk" - -# Reset the working directory -WORKDIR / - -# Define the default command. -CMD ["bash"] diff --git a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile index af2ab90b76..96154c8d8f 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharp/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile index af2ab90b76..96154c8d8f 100644 --- a/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_csharpcoreclr/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile index ec5338b4fc..3e0f477cc6 100644 --- a/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_cxx/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile index a775961ff7..83f05d87ea 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile index c296c8c674..79907f3d7c 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.7/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile index 9ac0b97341..42069419f8 100644 --- a/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_go1.8/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile index 92a2faa605..c14bcdaa93 100644 --- a/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_http2/Dockerfile @@ -28,7 +28,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile index 34b4e39df4..510e157d56 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java/Dockerfile @@ -43,7 +43,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile index 34b4e39df4..510e157d56 100644 --- a/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_java_oracle8/Dockerfile @@ -43,7 +43,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile index b480d31da2..7464e31f0f 100644 --- a/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_node/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile index 07f419f880..49c1191ba9 100644 --- a/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_python/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile index 543e207c64..f3667b5ae0 100644 --- a/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile +++ b/tools/dockerfile/interoptest/grpc_interop_ruby/Dockerfile @@ -60,7 +60,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile index 0fae21de63..c9335f0941 100644 --- a/tools/dockerfile/test/csharp_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/csharp_jessie_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile index c68a5ddbde..5b2b527a81 100644 --- a/tools/dockerfile/test/cxx_alpine_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_alpine_x64/Dockerfile @@ -32,12 +32,13 @@ RUN apk update && apk add \ strace \ python-dev \ py-pip \ + py-yaml \ unzip \ wget \ zip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile index dff4c96e9a..825896b1a5 100644 --- a/tools/dockerfile/test/cxx_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile index ea1d645628..692960dff7 100644 --- a/tools/dockerfile/test/cxx_jessie_x86/Dockerfile +++ b/tools/dockerfile/test/cxx_jessie_x86/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile index 990dac9128..38b040b258 100644 --- a/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1404_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile index c8943cada9..5487b6d28c 100644 --- a/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1604_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile b/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile index 11aca1766e..ffb08588cb 100644 --- a/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile +++ b/tools/dockerfile/test/cxx_ubuntu1710_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/fuzzer/Dockerfile b/tools/dockerfile/test/fuzzer/Dockerfile index 52666eab99..36712191ec 100644 --- a/tools/dockerfile/test/fuzzer/Dockerfile +++ b/tools/dockerfile/test/fuzzer/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile index 5818ff9fb4..8202a4a060 100644 --- a/tools/dockerfile/test/multilang_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/multilang_jessie_x64/Dockerfile @@ -140,7 +140,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/node_jessie_x64/Dockerfile b/tools/dockerfile/test/node_jessie_x64/Dockerfile index 05c8319495..e87eb6f63b 100644 --- a/tools/dockerfile/test/node_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/node_jessie_x64/Dockerfile @@ -75,7 +75,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/php7_jessie_x64/Dockerfile b/tools/dockerfile/test/php7_jessie_x64/Dockerfile index 865bf695fe..3a63867867 100644 --- a/tools/dockerfile/test/php7_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php7_jessie_x64/Dockerfile @@ -75,7 +75,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/php_jessie_x64/Dockerfile b/tools/dockerfile/test/php_jessie_x64/Dockerfile index fb653d365a..32a4b61a40 100644 --- a/tools/dockerfile/test/php_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/php_jessie_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/python_alpine_x64/Dockerfile b/tools/dockerfile/test/python_alpine_x64/Dockerfile index 8ae4550c4f..66ec34ab59 100644 --- a/tools/dockerfile/test/python_alpine_x64/Dockerfile +++ b/tools/dockerfile/test/python_alpine_x64/Dockerfile @@ -37,7 +37,7 @@ RUN apk update && apk add \ zip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 diff --git a/tools/dockerfile/test/python_jessie_x64/Dockerfile b/tools/dockerfile/test/python_jessie_x64/Dockerfile index 914e343373..4e6916d3c2 100644 --- a/tools/dockerfile/test/python_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/python_jessie_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/python_pyenv_x64/Dockerfile b/tools/dockerfile/test/python_pyenv_x64/Dockerfile index 379f26ad91..bd1432eaf7 100644 --- a/tools/dockerfile/test/python_pyenv_x64/Dockerfile +++ b/tools/dockerfile/test/python_pyenv_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile index 63e42fd2da..45923ca240 100644 --- a/tools/dockerfile/test/ruby_jessie_x64/Dockerfile +++ b/tools/dockerfile/test/ruby_jessie_x64/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/dockerfile/test/sanity/Dockerfile b/tools/dockerfile/test/sanity/Dockerfile index f65adf589d..c93693c658 100644 --- a/tools/dockerfile/test/sanity/Dockerfile +++ b/tools/dockerfile/test/sanity/Dockerfile @@ -64,7 +64,7 @@ RUN apt-get update && apt-get install -y \ python-pip # Install Python packages from PyPI -RUN pip install --upgrade pip==9.0.1 +RUN pip install --upgrade pip==9.0.2 RUN pip install virtualenv RUN pip install futures==2.2.0 enum34==1.0.4 protobuf==3.5.0.post1 six==1.10.0 twisted==17.5.0 diff --git a/tools/doxygen/Doxyfile.c++ b/tools/doxygen/Doxyfile.c++ index eb6700d529..22f225ec54 100644 --- a/tools/doxygen/Doxyfile.c++ +++ b/tools/doxygen/Doxyfile.c++ @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.11.0-dev +PROJECT_NUMBER = 1.12.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -954,6 +954,8 @@ include/grpcpp/impl/codegen/create_auth_context.h \ include/grpcpp/impl/codegen/grpc_library.h \ include/grpcpp/impl/codegen/metadata_map.h \ include/grpcpp/impl/codegen/method_handler_impl.h \ +include/grpcpp/impl/codegen/proto_buffer_reader.h \ +include/grpcpp/impl/codegen/proto_buffer_writer.h \ include/grpcpp/impl/codegen/proto_utils.h \ include/grpcpp/impl/codegen/rpc_method.h \ include/grpcpp/impl/codegen/rpc_service_method.h \ @@ -992,6 +994,8 @@ include/grpcpp/support/async_unary_call.h \ include/grpcpp/support/byte_buffer.h \ include/grpcpp/support/channel_arguments.h \ include/grpcpp/support/config.h \ +include/grpcpp/support/proto_buffer_reader.h \ +include/grpcpp/support/proto_buffer_writer.h \ include/grpcpp/support/slice.h \ include/grpcpp/support/status.h \ include/grpcpp/support/status_code_enum.h \ diff --git a/tools/doxygen/Doxyfile.c++.internal b/tools/doxygen/Doxyfile.c++.internal index 52a16d6e98..63d238d741 100644 --- a/tools/doxygen/Doxyfile.c++.internal +++ b/tools/doxygen/Doxyfile.c++.internal @@ -40,7 +40,7 @@ PROJECT_NAME = "GRPC C++" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = 1.11.0-dev +PROJECT_NUMBER = 1.12.0-dev # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a @@ -956,6 +956,8 @@ include/grpcpp/impl/codegen/create_auth_context.h \ include/grpcpp/impl/codegen/grpc_library.h \ include/grpcpp/impl/codegen/metadata_map.h \ include/grpcpp/impl/codegen/method_handler_impl.h \ +include/grpcpp/impl/codegen/proto_buffer_reader.h \ +include/grpcpp/impl/codegen/proto_buffer_writer.h \ include/grpcpp/impl/codegen/proto_utils.h \ include/grpcpp/impl/codegen/rpc_method.h \ include/grpcpp/impl/codegen/rpc_service_method.h \ @@ -994,6 +996,8 @@ include/grpcpp/support/async_unary_call.h \ include/grpcpp/support/byte_buffer.h \ include/grpcpp/support/channel_arguments.h \ include/grpcpp/support/config.h \ +include/grpcpp/support/proto_buffer_reader.h \ +include/grpcpp/support/proto_buffer_writer.h \ include/grpcpp/support/slice.h \ include/grpcpp/support/status.h \ include/grpcpp/support/status_code_enum.h \ @@ -1209,7 +1213,6 @@ src/cpp/server/thread_pool_interface.h \ src/cpp/thread_manager/thread_manager.cc \ src/cpp/thread_manager/thread_manager.h \ src/cpp/util/byte_buffer_cc.cc \ -src/cpp/util/slice_cc.cc \ src/cpp/util/status.cc \ src/cpp/util/string_ref.cc \ src/cpp/util/time_cc.cc \ diff --git a/tools/gce/linux_kokoro_performance_worker_init.sh b/tools/gce/linux_kokoro_performance_worker_init.sh index 1f98d24595..084e9fbff4 100755 --- a/tools/gce/linux_kokoro_performance_worker_init.sh +++ b/tools/gce/linux_kokoro_performance_worker_init.sh @@ -72,7 +72,7 @@ sudo apt-get install -y netperf sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang # Python dependencies -sudo pip install --upgrade pip==9.0.1 +sudo pip install --upgrade pip==9.0.2 sudo pip install tabulate sudo pip install google-api-python-client sudo pip install virtualenv diff --git a/tools/gce/linux_performance_worker_init.sh b/tools/gce/linux_performance_worker_init.sh index 6f6866096d..09d8fd012d 100755 --- a/tools/gce/linux_performance_worker_init.sh +++ b/tools/gce/linux_performance_worker_init.sh @@ -72,7 +72,7 @@ sudo apt-get install -y netperf sudo apt-get install -y libgflags-dev libgtest-dev libc++-dev clang # Python dependencies -sudo pip install --upgrade pip==9.0.1 +sudo pip install --upgrade pip==9.0.2 sudo pip install tabulate sudo pip install google-api-python-client sudo pip install virtualenv diff --git a/tools/internal_ci/helper_scripts/gen_report_index.sh b/tools/internal_ci/helper_scripts/gen_report_index.sh deleted file mode 100755 index 576ff67d54..0000000000 --- a/tools/internal_ci/helper_scripts/gen_report_index.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Generates index.html that will contain links to various test results on kokoro. -set -e - -# change to grpc repo root -cd $(dirname $0)/../../.. - -# Kororo URLs are in the form "grpc/job/macos/job/master/job/grpc_build_artifacts" -KOKORO_JOB_PATH=$(echo "${KOKORO_JOB_NAME}" | sed "s|/|/job/|g") - -mkdir -p reports - -echo '<html><head></head><body>' > reports/kokoro_index.html -echo '<h1>'${KOKORO_JOB_NAME}', build '#${KOKORO_BUILD_NUMBER}'</h1>' >> reports/kokoro_index.html -echo '<h2><a href="https://kokoro2.corp.google.com/job/'${KOKORO_JOB_PATH}'/'${KOKORO_BUILD_NUMBER}'/">Kokoro build dashboard (internal only)</a></h2>' >> reports/kokoro_index.html -echo '<h2><a href="https://sponge.corp.google.com/invocation?id='${KOKORO_BUILD_ID}'&searchFor=">Test result dashboard (internal only)</a></h2>' >> reports/kokoro_index.html -echo '<h2><a href="test_report.html">HTML test report (Not available yet)</a></h2>' >> reports/kokoro_index.html -echo '<h2><a href="test_log.txt">Test log (Not available yet)</a></h2>' >> reports/kokoro_index.html -echo '</body></html>' >> reports/kokoro_index.html - -echo 'Created reports/kokoro_index.html report index' diff --git a/tools/internal_ci/helper_scripts/prepare_build_linux_rc b/tools/internal_ci/helper_scripts/prepare_build_linux_rc index 74bbc85d72..992c54008c 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_linux_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_linux_rc @@ -15,8 +15,6 @@ # Source this rc script to prepare the environment for linux builds -tools/internal_ci/helper_scripts/gen_report_index.sh - # Need to increase open files limit for c tests ulimit -n 32768 diff --git a/tools/internal_ci/helper_scripts/prepare_build_macos_rc b/tools/internal_ci/helper_scripts/prepare_build_macos_rc index 3a09701a30..47c1d4b103 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_macos_rc +++ b/tools/internal_ci/helper_scripts/prepare_build_macos_rc @@ -15,8 +15,6 @@ # Source this rc script to prepare the environment for macos builds -tools/internal_ci/helper_scripts/gen_report_index.sh - sudo launchctl limit maxfiles unlimited unlimited # show current maxfiles diff --git a/tools/internal_ci/helper_scripts/prepare_build_windows.bat b/tools/internal_ci/helper_scripts/prepare_build_windows.bat index f625755db9..9fa6860226 100644 --- a/tools/internal_ci/helper_scripts/prepare_build_windows.bat +++ b/tools/internal_ci/helper_scripts/prepare_build_windows.bat @@ -16,8 +16,6 @@ @rem set path to python 2.7 set PATH=C:\tools\msys64\usr\bin;C:\Python27;%PATH% -bash tools/internal_ci/helper_scripts/gen_report_index.sh - @rem If this is a PR using RUN_TESTS_FLAGS var, then add flags to filter tests if defined KOKORO_GITHUB_PULL_REQUEST_NUMBER if defined RUN_TESTS_FLAGS ( chocolatey install -y jq diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh index f5111e2feb..3ff569df47 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_dbg.sh @@ -13,6 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,7200" +EXTRA_FLAGS="-c dbg --test_timeout=300,450,1200,3600" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh index 7f3e4cf8b5..ded0d368a5 100644 --- a/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh +++ b/tools/internal_ci/linux/grpc_bazel_on_foundry_opt.sh @@ -13,5 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,7200" +EXTRA_FLAGS="-c opt --test_timeout=300,450,1200,3600" github/grpc/tools/internal_ci/linux/grpc_bazel_on_foundry_base.sh "${EXTRA_FLAGS}" diff --git a/tools/internal_ci/linux/grpc_build_packages.sh b/tools/internal_ci/linux/grpc_build_packages.sh index 9f65d8ed5f..ff71298add 100644 --- a/tools/internal_ci/linux/grpc_build_packages.sh +++ b/tools/internal_ci/linux/grpc_build_packages.sh @@ -30,6 +30,7 @@ set -ex # where they can be accessed from within a docker container that builds # the packages mv ${KOKORO_GFILE_DIR}/github/grpc/artifacts input_artifacts || true +chmod +x input_artifacts/protoc*/* || true ls -R input_artifacts || true tools/run_tests/task_runner.py -f package linux -j 6 diff --git a/tools/internal_ci/linux/grpc_coverage.cfg b/tools/internal_ci/linux/grpc_coverage.cfg index 56b7745b21..794a51d3f1 100644 --- a/tools/internal_ci/linux/grpc_coverage.cfg +++ b/tools/internal_ci/linux/grpc_coverage.cfg @@ -19,6 +19,7 @@ build_file: "grpc/tools/internal_ci/linux/grpc_coverage.sh" timeout_mins: 420 action { define_artifacts { + regex: "**/*sponge_log.xml" regex: "github/grpc/reports/**" } } diff --git a/tools/internal_ci/linux/grpc_coverage.sh b/tools/internal_ci/linux/grpc_coverage.sh index 12181f0133..fd927a85fd 100755 --- a/tools/internal_ci/linux/grpc_coverage.sh +++ b/tools/internal_ci/linux/grpc_coverage.sh @@ -21,10 +21,9 @@ cd $(dirname $0)/../../.. source tools/internal_ci/helper_scripts/prepare_build_linux_rc python tools/run_tests/run_tests.py \ - --use_docker \ + --use_docker \ -t \ -l all \ -c gcov \ - -x report.xml \ + -x sponge_log.xml \ -j 16 - diff --git a/tools/internal_ci/linux/grpc_interop_alts.cfg b/tools/internal_ci/linux/grpc_interop_alts.cfg new file mode 100644 index 0000000000..bda76faf44 --- /dev/null +++ b/tools/internal_ci/linux/grpc_interop_alts.cfg @@ -0,0 +1,30 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" +timeout_mins: 60 +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-l all -s all --use_docker --transport_security alts --internal_ci -t -j 12 --bq_result_table interop_results" +} diff --git a/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg b/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg new file mode 100644 index 0000000000..c1253b30f7 --- /dev/null +++ b/tools/internal_ci/linux/pull_request/grpc_interop_alts.cfg @@ -0,0 +1,30 @@ +# Copyright 2018 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/linux/grpc_run_interop_tests.sh" +timeout_mins: 60 +action { + define_artifacts { + regex: "**/sponge_log.xml" + regex: "github/grpc/reports/**" + } +} + +env_vars { + key: "RUN_TESTS_FLAGS" + value: "-l all -s all --use_docker --transport_security alts --internal_ci -t -j 12" +} diff --git a/tools/internal_ci/macos/grpc_interop_toprod.cfg b/tools/internal_ci/macos/grpc_interop_toprod.cfg new file mode 100644 index 0000000000..c92c397daa --- /dev/null +++ b/tools/internal_ci/macos/grpc_interop_toprod.cfg @@ -0,0 +1,27 @@ +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Config file for the internal CI (in protobuf text format) + +# Location of the continuous shell script in repository. +build_file: "grpc/tools/internal_ci/macos/grpc_interop_toprod.sh" +gfile_resources: "/bigstore/grpc-testing-secrets/gcp_credentials/GrpcTesting-d0eeee2db331.json" +gfile_resources: "/bigstore/grpc-testing-secrets/interop/service_account/GrpcTesting-726eb1347f15.json" +timeout_mins: 240 +action { + define_artifacts { + regex: "**/*sponge_log.xml" + regex: "github/grpc/reports/**" + } +} diff --git a/tools/internal_ci/macos/grpc_interop_toprod.sh b/tools/internal_ci/macos/grpc_interop_toprod.sh new file mode 100755 index 0000000000..819a4721fd --- /dev/null +++ b/tools/internal_ci/macos/grpc_interop_toprod.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2017 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# change to grpc repo root +cd $(dirname $0)/../../.. + +source tools/internal_ci/helper_scripts/prepare_build_macos_interop_rc +source tools/internal_ci/helper_scripts/prepare_build_macos_rc + +# using run_interop_tests.py without --use_docker, so we need to build first +tools/run_tests/run_tests.py -l c++ -c opt --build_only + +export GRPC_DEFAULT_SSL_ROOTS_FILE_PATH="$(pwd)/etc/roots.pem" + +# NOTE: only tests a subset of languages for time & dependency constraints +# building all languages in the same working copy can also lead to conflicts +# due to different compilation flags +tools/run_tests/run_interop_tests.py -l c++ \ + --cloud_to_prod --cloud_to_prod_auth --prod_servers default gateway_v4 \ + --service_account_key_file="${KOKORO_GFILE_DIR}/GrpcTesting-726eb1347f15.json" \ + --skip_compute_engine_creds --internal_ci -t -j 4 diff --git a/tools/interop_matrix/client_matrix.py b/tools/interop_matrix/client_matrix.py index c45a627395..7b4b23425e 100644 --- a/tools/interop_matrix/client_matrix.py +++ b/tools/interop_matrix/client_matrix.py @@ -120,7 +120,7 @@ LANG_RELEASE_MATRIX = { 'v1.10.1': None }, { - 'v1.11.1': None + 'v1.11.3': None }, ], 'java': [ diff --git a/tools/run_tests/README.md b/tools/run_tests/README.md index 60f20747cd..ceddc21a98 100644 --- a/tools/run_tests/README.md +++ b/tools/run_tests/README.md @@ -14,6 +14,10 @@ Builds gRPC in given language and runs unit tests. Use `tools/run_tests/run_test - `--use_docker` Builds a docker container containing all the prerequisites for given language and runs the tests under that container. - `--build_only` Only build, do not run the tests. +Note: some tests may be flaky. Check the "Issues" tab for known flakes and other issues. + +The full suite of unit tests will take many minutes to run. + # Interop tests (run_interop_tests.py) Runs tests for cross-platform/cross-language interoperability. For more details, see [Interop tests descriptions](/doc/interop-test-descriptions.md) diff --git a/tools/run_tests/artifacts/build_package_ruby.sh b/tools/run_tests/artifacts/build_package_ruby.sh index 05eacd4d1e..4f74f08276 100755 --- a/tools/run_tests/artifacts/build_package_ruby.sh +++ b/tools/run_tests/artifacts/build_package_ruby.sh @@ -53,8 +53,11 @@ for arch in {x86,x64}; do output_dir="$base/src/ruby/tools/bin/${ruby_arch}-${plat}" mkdir -p "$output_dir"/google/protobuf mkdir -p "$output_dir"/google/protobuf/compiler # needed for plugin.proto - cp "$input_dir"/protoc* "$output_dir"/ - cp "$input_dir"/grpc_ruby_plugin* "$output_dir"/ + cp "$input_dir"/protoc* "$input_dir"/grpc_ruby_plugin* "$output_dir/" + if [[ "$plat" != "windows" ]] + then + chmod +x "$output_dir/protoc" "$output_dir/grpc_ruby_plugin" + fi for proto in "${well_known_protos[@]}"; do cp "$base/third_party/protobuf/src/google/protobuf/$proto.proto" "$output_dir/google/protobuf/$proto.proto" done diff --git a/tools/run_tests/generated/sources_and_headers.json b/tools/run_tests/generated/sources_and_headers.json index 9676fe379d..ebbfdbf230 100644 --- a/tools/run_tests/generated/sources_and_headers.json +++ b/tools/run_tests/generated/sources_and_headers.json @@ -10745,6 +10745,8 @@ ], "headers": [ "include/grpc++/impl/codegen/proto_utils.h", + "include/grpcpp/impl/codegen/proto_buffer_reader.h", + "include/grpcpp/impl/codegen/proto_buffer_writer.h", "include/grpcpp/impl/codegen/proto_utils.h" ], "is_filegroup": true, @@ -10752,6 +10754,8 @@ "name": "grpc++_codegen_proto", "src": [ "include/grpc++/impl/codegen/proto_utils.h", + "include/grpcpp/impl/codegen/proto_buffer_reader.h", + "include/grpcpp/impl/codegen/proto_buffer_writer.h", "include/grpcpp/impl/codegen/proto_utils.h" ], "third_party": false, @@ -10850,6 +10854,8 @@ "include/grpcpp/support/byte_buffer.h", "include/grpcpp/support/channel_arguments.h", "include/grpcpp/support/config.h", + "include/grpcpp/support/proto_buffer_reader.h", + "include/grpcpp/support/proto_buffer_writer.h", "include/grpcpp/support/slice.h", "include/grpcpp/support/status.h", "include/grpcpp/support/status_code_enum.h", @@ -10952,6 +10958,8 @@ "include/grpcpp/support/byte_buffer.h", "include/grpcpp/support/channel_arguments.h", "include/grpcpp/support/config.h", + "include/grpcpp/support/proto_buffer_reader.h", + "include/grpcpp/support/proto_buffer_writer.h", "include/grpcpp/support/slice.h", "include/grpcpp/support/status.h", "include/grpcpp/support/status_code_enum.h", @@ -10996,7 +11004,6 @@ "src/cpp/thread_manager/thread_manager.cc", "src/cpp/thread_manager/thread_manager.h", "src/cpp/util/byte_buffer_cc.cc", - "src/cpp/util/slice_cc.cc", "src/cpp/util/status.cc", "src/cpp/util/string_ref.cc", "src/cpp/util/time_cc.cc" diff --git a/tools/run_tests/helper_scripts/build_python.sh b/tools/run_tests/helper_scripts/build_python.sh index dac29b91dd..0b5d3316ad 100755 --- a/tools/run_tests/helper_scripts/build_python.sh +++ b/tools/run_tests/helper_scripts/build_python.sh @@ -132,7 +132,7 @@ fi # Perform build operations # ############################ -# Instnatiate the virtualenv, preferring to do so from the relevant python +# Instantiate the virtualenv, preferring to do so from the relevant python # version. Even if these commands fail (e.g. on Windows due to name conflicts) # it's possible that the virtualenv is still usable and we trust the tester to # be able to 'figure it out' instead of us e.g. doing potentially expensive and @@ -158,7 +158,7 @@ case "$VENV" in ;; esac -$VENV_PYTHON -m pip install --upgrade pip==9.0.1 +$VENV_PYTHON -m pip install --upgrade pip==9.0.2 $VENV_PYTHON -m pip install setuptools $VENV_PYTHON -m pip install cython $VENV_PYTHON -m pip install six enum34 protobuf futures diff --git a/tools/run_tests/interop/android/android_interop_helper.sh b/tools/run_tests/interop/android/android_interop_helper.sh deleted file mode 100755 index 116549b83e..0000000000 --- a/tools/run_tests/interop/android/android_interop_helper.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Helper that runs inside the docker container and builds the APKs and -# invokes Firebase Test Lab via gcloud. - -SERVICE_KEY=$1 - -gcloud auth activate-service-account --key-file="$SERVICE_KEY" || exit 1 -gcloud config set project grpc-testing || exit 1 - -rm -rf grpc-java -git clone https://github.com/grpc/grpc-java.git -cd grpc-java -./gradlew install || exit 1 -cd android-interop-testing -../gradlew assembleDebug -../gradlew assembleDebugAndroidTest - -gcloud firebase test android run \ - --type instrumentation \ - --app app/build/outputs/apk/app-debug.apk \ - --test app/build/outputs/apk/app-debug-androidTest.apk \ - --device model=Nexus6,version=21,locale=en,orientation=portrait diff --git a/tools/run_tests/interop/android/run_android_tests_on_firebase.sh b/tools/run_tests/interop/android/run_android_tests_on_firebase.sh deleted file mode 100755 index f6472eba55..0000000000 --- a/tools/run_tests/interop/android/run_android_tests_on_firebase.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash -# Copyright 2017 gRPC authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Builds the gRPC Android instrumented interop tests inside a docker container -# and runs them on Firebase Test Lab - -DOCKERFILE=tools/dockerfile/interoptest/grpc_interop_android_java/Dockerfile -DOCKER_TAG=android_interop_test -SERVICE_KEY=~/android-interops-service-key.json -HELPER=$(pwd)/tools/run_tests/interop/android/android_interop_helper.sh - -docker build -t "$DOCKER_TAG" -f "$DOCKERFILE" . - -docker run --interactive --rm \ - --volume="$SERVICE_KEY":/service-key.json:ro \ - --volume="$HELPER":/android_interop_helper.sh:ro \ - $DOCKER_TAG \ - /bin/bash -c "/android_interop_helper.sh /service-key.json" diff --git a/tools/run_tests/python_utils/upload_test_results.py b/tools/run_tests/python_utils/upload_test_results.py index a2dd1c66cf..09dcd57ad4 100644 --- a/tools/run_tests/python_utils/upload_test_results.py +++ b/tools/run_tests/python_utils/upload_test_results.py @@ -74,7 +74,7 @@ def _get_build_metadata(test_results): build_id = os.getenv('BUILD_ID') or os.getenv('KOKORO_BUILD_NUMBER') build_url = os.getenv('BUILD_URL') if os.getenv('KOKORO_BUILD_ID'): - build_url = 'https://sponge.corp.google.com/invocation?id=%s' % os.getenv( + build_url = 'https://source.cloud.google.com/results/invocations/%s' % os.getenv( 'KOKORO_BUILD_ID') job_name = os.getenv('JOB_BASE_NAME') or os.getenv('KOKORO_JOB_NAME') diff --git a/tools/run_tests/run_interop_tests.py b/tools/run_tests/run_interop_tests.py index 970b0d7902..56aee6419e 100755 --- a/tools/run_tests/run_interop_tests.py +++ b/tools/run_tests/run_interop_tests.py @@ -637,6 +637,14 @@ _LANGUAGES_WITH_HTTP2_CLIENTS_FOR_HTTP2_SERVER_TEST_CASES = [ 'java', 'go', 'python', 'c++' ] +#TODO: Add c++ when c++ ALTS interop client is ready. +_LANGUAGES_FOR_ALTS_TEST_CASES = ['java', 'go'] + +#TODO: Add c++ when c++ ALTS interop server is ready. +_SERVERS_FOR_ALTS_TEST_CASES = ['java', 'go'] + +_TRANSPORT_SECURITY_OPTIONS = ['tls', 'alts', 'insecure'] + DOCKER_WORKDIR_ROOT = '/var/local/git/grpc' @@ -691,17 +699,29 @@ def bash_cmdline(cmdline): return ['bash', '-c', ' '.join(cmdline)] -def auth_options(language, test_case): +def compute_engine_creds_required(language, test_case): + """Returns True if given test requires access to compute engine creds.""" + language = str(language) + if test_case == 'compute_engine_creds': + return True + if test_case == 'oauth2_auth_token' and language == 'c++': + # C++ oauth2 test uses GCE creds because C++ only supports JWT + return True + return False + + +def auth_options(language, test_case, service_account_key_file=None): """Returns (cmdline, env) tuple with cloud_to_prod_auth test options.""" language = str(language) cmdargs = [] env = {} - # TODO(jtattermusch): this file path only works inside docker - key_filepath = '/root/service_account/GrpcTesting-726eb1347f15.json' + if not service_account_key_file: + # this file path only works inside docker + service_account_key_file = '/root/service_account/GrpcTesting-726eb1347f15.json' oauth_scope_arg = '--oauth_scope=https://www.googleapis.com/auth/xapi.zoo' - key_file_arg = '--service_account_key_file=%s' % key_filepath + key_file_arg = '--service_account_key_file=%s' % service_account_key_file default_account_arg = '--default_service_account=830293263384-compute@developer.gserviceaccount.com' if test_case in ['jwt_token_creds', 'per_rpc_creds', 'oauth2_auth_token']: @@ -709,7 +729,7 @@ def auth_options(language, test_case): 'csharp', 'csharpcoreclr', 'node', 'php', 'php7', 'python', 'ruby', 'nodepurejs' ]: - env['GOOGLE_APPLICATION_CREDENTIALS'] = key_filepath + env['GOOGLE_APPLICATION_CREDENTIALS'] = service_account_key_file else: cmdargs += [key_file_arg] @@ -742,7 +762,8 @@ def cloud_to_prod_jobspec(language, server_host_detail, docker_image=None, auth=False, - manual_cmd_log=None): + manual_cmd_log=None, + service_account_key_file=None): """Creates jobspec for cloud-to-prod interop test""" container_name = None cmdargs = [ @@ -753,7 +774,8 @@ def cloud_to_prod_jobspec(language, ] environ = dict(language.cloud_to_prod_env(), **language.global_env()) if auth: - auth_cmdargs, auth_env = auth_options(language, test_case) + auth_cmdargs, auth_env = auth_options(language, test_case, + service_account_key_file) cmdargs += auth_cmdargs environ.update(auth_env) cmdline = bash_cmdline(language.client_cmd(cmdargs)) @@ -799,14 +821,22 @@ def cloud_to_cloud_jobspec(language, server_host, server_port, docker_image=None, - insecure=False, + transport_security='tls', manual_cmd_log=None): """Creates jobspec for cloud-to-cloud interop test""" interop_only_options = [ '--server_host_override=foo.test.google.fr', - '--use_tls=%s' % ('false' if insecure else 'true'), '--use_test_ca=true', ] + if transport_security == 'tls': + interop_only_options += ['--use_tls=true'] + elif transport_security == 'alts': + interop_only_options += ['--use_tls=false', '--use_alts=true'] + elif transport_security == 'insecure': + interop_only_options += ['--use_tls=false'] + else: + print('Invalid transport security option.') + sys.exit(1) client_test_case = test_case if test_case in _HTTP2_SERVER_TEST_CASES_THAT_USE_GRPC_CLIENTS: @@ -871,15 +901,24 @@ def cloud_to_cloud_jobspec(language, return test_job -def server_jobspec(language, docker_image, insecure=False, manual_cmd_log=None): +def server_jobspec(language, + docker_image, + transport_security='tls', + manual_cmd_log=None): """Create jobspec for running a server""" container_name = dockerjob.random_name( 'interop_server_%s' % language.safename) - cmdline = bash_cmdline( - language.server_cmd([ - '--port=%s' % _DEFAULT_SERVER_PORT, - '--use_tls=%s' % ('false' if insecure else 'true') - ])) + server_cmd = ['--port=%s' % _DEFAULT_SERVER_PORT] + if transport_security == 'tls': + server_cmd += ['--use_tls=true'] + elif transport_security == 'alts': + server_cmd += ['--use_tls=false', '--use_alts=true'] + elif transport_security == 'insecure': + server_cmd += ['--use_tls=false'] + else: + print('Invalid transport security option.') + sys.exit(1) + cmdline = bash_cmdline(language.server_cmd(server_cmd)) environ = language.global_env() docker_args = ['--name=%s' % container_name] if language.safename == 'http2': @@ -1044,6 +1083,12 @@ argp.add_argument( 'Use servername=HOST:PORT to explicitly specify a server. E.g. csharp=localhost:50000', default=[]) argp.add_argument( + '--service_account_key_file', + type=str, + help= + 'Override the default service account key file to use for auth interop tests.', + default=None) +argp.add_argument( '-t', '--travis', default=False, action='store_const', const=True) argp.add_argument( '-v', '--verbose', default=False, action='store_const', const=True) @@ -1086,11 +1131,19 @@ argp.add_argument( 'Enable HTTP/2 server edge case testing. (Includes positive and negative tests' ) argp.add_argument( - '--insecure', + '--transport_security', + choices=_TRANSPORT_SECURITY_OPTIONS, + default='tls', + type=str, + nargs='?', + const=True, + help='Which transport security mechanism to use.') +argp.add_argument( + '--skip_compute_engine_creds', default=False, action='store_const', const=True, - help='Whether to use secure channel.') + help='Skip auth tests requiring access to compute engine credentials.') argp.add_argument( '--internal_ci', default=False, @@ -1110,6 +1163,9 @@ servers = set( s for s in itertools.chain.from_iterable( _SERVERS if x == 'all' else [x] for x in args.server)) +# ALTS servers are only available for certain languages. +if args.transport_security == 'alts': + servers = servers.intersection(_SERVERS_FOR_ALTS_TEST_CASES) if args.use_docker: if not args.travis: @@ -1139,6 +1195,10 @@ all_but_objc = set(six.iterkeys(_LANGUAGES)) - set(['objc']) languages = set(_LANGUAGES[l] for l in itertools.chain.from_iterable( all_but_objc if x == 'all' else [x] for x in args.language)) +# ALTS interop clients are only available for certain languages. +if args.transport_security == 'alts': + alts_languages = set(_LANGUAGES[l] for l in _LANGUAGES_FOR_ALTS_TEST_CASES) + languages = languages.intersection(alts_languages) languages_http2_clients_for_http2_server_interop = set() if args.http2_server_interop: @@ -1207,7 +1267,7 @@ try: spec = server_jobspec( _LANGUAGES[lang], docker_images.get(lang), - args.insecure, + args.transport_security, manual_cmd_log=server_manual_cmd_log) if not args.manual_run: job = dockerjob.DockerJob(spec) @@ -1235,7 +1295,7 @@ try: jobs = [] if args.cloud_to_prod: - if args.insecure: + if args.transport_security != 'tls': print('TLS is always enabled for cloud_to_prod scenarios.') for server_host_name in args.prod_servers: for language in languages: @@ -1248,7 +1308,9 @@ try: server_host_name, prod_servers[server_host_name], docker_image=docker_images.get(str(language)), - manual_cmd_log=client_manual_cmd_log) + manual_cmd_log=client_manual_cmd_log, + service_account_key_file=args. + service_account_key_file) jobs.append(test_job) if args.http2_interop: @@ -1259,25 +1321,31 @@ try: server_host_name, prod_servers[server_host_name], docker_image=docker_images.get(str(http2Interop)), - manual_cmd_log=client_manual_cmd_log) + manual_cmd_log=client_manual_cmd_log, + service_account_key_file=args.service_account_key_file) jobs.append(test_job) if args.cloud_to_prod_auth: - if args.insecure: + if args.transport_security != 'tls': print('TLS is always enabled for cloud_to_prod scenarios.') for server_host_name in args.prod_servers: for language in languages: for test_case in _AUTH_TEST_CASES: - if not test_case in language.unimplemented_test_cases(): - test_job = cloud_to_prod_jobspec( - language, - test_case, - server_host_name, - prod_servers[server_host_name], - docker_image=docker_images.get(str(language)), - auth=True, - manual_cmd_log=client_manual_cmd_log) - jobs.append(test_job) + if (not args.skip_compute_engine_creds or + not compute_engine_creds_required( + language, test_case)): + if not test_case in language.unimplemented_test_cases(): + test_job = cloud_to_prod_jobspec( + language, + test_case, + server_host_name, + prod_servers[server_host_name], + docker_image=docker_images.get(str(language)), + auth=True, + manual_cmd_log=client_manual_cmd_log, + service_account_key_file=args. + service_account_key_file) + jobs.append(test_job) for server in args.override_server: server_name = server[0] @@ -1301,7 +1369,7 @@ try: server_host, server_port, docker_image=docker_images.get(str(language)), - insecure=args.insecure, + transport_security=args.transport_security, manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) @@ -1317,7 +1385,7 @@ try: server_host, server_port, docker_image=docker_images.get(str(http2Interop)), - insecure=args.insecure, + transport_security=args.transport_security, manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) @@ -1353,11 +1421,12 @@ try: server_port = _DEFAULT_SERVER_PORT + offset if not args.manual_run: server_port = http2_server_job.mapped_port(server_port) - if not args.insecure: - print(( - 'Creating grpc cient to http2 server test case with insecure connection, even though' - ' args.insecure is False. Http2 test server only supports insecure connections.' - )) + if args.transport_security != 'insecure': + print( + ('Creating grpc client to http2 server test case ' + 'with insecure connection, even though ' + 'args.transport_security is not insecure. Http2 ' + 'test server only supports insecure connections.')) test_job = cloud_to_cloud_jobspec( language, test_case, @@ -1365,7 +1434,7 @@ try: 'localhost', server_port, docker_image=docker_images.get(str(language)), - insecure=True, + transport_security='insecure', manual_cmd_log=client_manual_cmd_log) jobs.append(test_job) |