diff options
Diffstat (limited to 'grpc/src/core/ext/xds')
22 files changed, 4458 insertions, 1364 deletions
diff --git a/grpc/src/core/ext/xds/certificate_provider_factory.h b/grpc/src/core/ext/xds/certificate_provider_factory.h index 84c219e6..e9bba790 100644 --- a/grpc/src/core/ext/xds/certificate_provider_factory.h +++ b/grpc/src/core/ext/xds/certificate_provider_factory.h @@ -49,7 +49,7 @@ class CertificateProviderFactory { virtual const char* name() const = 0; virtual RefCountedPtr<Config> CreateCertificateProviderConfig( - const Json& config_json, grpc_error** error) = 0; + const Json& config_json, grpc_error_handle* error) = 0; // Create a CertificateProvider instance from config. virtual RefCountedPtr<grpc_tls_certificate_provider> diff --git a/grpc/src/core/ext/xds/certificate_provider_store.h b/grpc/src/core/ext/xds/certificate_provider_store.h index 0954bc5e..fb6ca72d 100644 --- a/grpc/src/core/ext/xds/certificate_provider_store.h +++ b/grpc/src/core/ext/xds/certificate_provider_store.h @@ -92,7 +92,7 @@ class CertificateProviderStore }; RefCountedPtr<CertificateProviderWrapper> CreateCertificateProviderLocked( - absl::string_view key); + absl::string_view key) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Releases a previously created certificate provider from the certificate // provider map if the value matches \a wrapper. @@ -101,10 +101,10 @@ class CertificateProviderStore Mutex mu_; // Map of plugin configurations - PluginDefinitionMap plugin_config_map_; + PluginDefinitionMap plugin_config_map_ ABSL_GUARDED_BY(mu_); // Underlying map for the providers. std::map<absl::string_view, CertificateProviderWrapper*> - certificate_providers_map_; + certificate_providers_map_ ABSL_GUARDED_BY(mu_); }; } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.cc b/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.cc index a5250eba..7a793b06 100644 --- a/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.cc +++ b/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.cc @@ -64,14 +64,14 @@ std::string FileWatcherCertificateProviderFactory::Config::ToString() const { RefCountedPtr<FileWatcherCertificateProviderFactory::Config> FileWatcherCertificateProviderFactory::Config::Parse(const Json& config_json, - grpc_error** error) { + grpc_error_handle* error) { auto config = MakeRefCounted<FileWatcherCertificateProviderFactory::Config>(); if (config_json.type() != Json::Type::OBJECT) { *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( "error:config type should be OBJECT."); return nullptr; } - std::vector<grpc_error*> error_list; + std::vector<grpc_error_handle> error_list; ParseJsonObjectField(config_json.object_value(), "certificate_file", &config->identity_cert_file_, &error_list, false); ParseJsonObjectField(config_json.object_value(), "private_key_file", @@ -112,7 +112,7 @@ const char* FileWatcherCertificateProviderFactory::name() const { RefCountedPtr<CertificateProviderFactory::Config> FileWatcherCertificateProviderFactory::CreateCertificateProviderConfig( - const Json& config_json, grpc_error** error) { + const Json& config_json, grpc_error_handle* error) { return FileWatcherCertificateProviderFactory::Config::Parse(config_json, error); } diff --git a/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.h b/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.h index c5700625..13e10deb 100644 --- a/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.h +++ b/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.h @@ -31,7 +31,7 @@ class FileWatcherCertificateProviderFactory class Config : public CertificateProviderFactory::Config { public: static RefCountedPtr<Config> Parse(const Json& config_json, - grpc_error** error); + grpc_error_handle* error); const char* name() const override; @@ -58,7 +58,7 @@ class FileWatcherCertificateProviderFactory RefCountedPtr<CertificateProviderFactory::Config> CreateCertificateProviderConfig(const Json& config_json, - grpc_error** error) override; + grpc_error_handle* error) override; RefCountedPtr<grpc_tls_certificate_provider> CreateCertificateProvider( RefCountedPtr<CertificateProviderFactory::Config> config) override; diff --git a/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.cc b/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.cc index c1b7b84a..6e63ae4e 100644 --- a/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.cc +++ b/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.cc @@ -52,10 +52,10 @@ std::string GoogleMeshCaCertificateProviderFactory::Config::ToString() const { return "{}"; } -std::vector<grpc_error*> +std::vector<grpc_error_handle> GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectStsService( const Json::Object& sts_service) { - std::vector<grpc_error*> error_list_sts_service; + std::vector<grpc_error_handle> error_list_sts_service; if (!ParseJsonObjectField(sts_service, "token_exchange_service_uri", &sts_config_.token_exchange_service_uri, &error_list_sts_service, false)) { @@ -89,14 +89,14 @@ GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectStsService( return error_list_sts_service; } -std::vector<grpc_error*> +std::vector<grpc_error_handle> GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectCallCredentials( const Json::Object& call_credentials) { - std::vector<grpc_error*> error_list_call_credentials; + std::vector<grpc_error_handle> error_list_call_credentials; const Json::Object* sts_service = nullptr; if (ParseJsonObjectField(call_credentials, "sts_service", &sts_service, &error_list_call_credentials)) { - std::vector<grpc_error*> error_list_sts_service = + std::vector<grpc_error_handle> error_list_sts_service = ParseJsonObjectStsService(*sts_service); if (!error_list_sts_service.empty()) { error_list_call_credentials.push_back(GRPC_ERROR_CREATE_FROM_VECTOR( @@ -106,10 +106,10 @@ GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectCallCredentials( return error_list_call_credentials; } -std::vector<grpc_error*> +std::vector<grpc_error_handle> GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectGoogleGrpc( const Json::Object& google_grpc) { - std::vector<grpc_error*> error_list_google_grpc; + std::vector<grpc_error_handle> error_list_google_grpc; if (!ParseJsonObjectField(google_grpc, "target_uri", &endpoint_, &error_list_google_grpc, false)) { endpoint_ = "meshca.googleapis.com"; // Default target @@ -124,7 +124,7 @@ GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectGoogleGrpc( const Json::Object* call_credentials = nullptr; if (ExtractJsonType((*call_credentials_array)[0], "call_credentials[0]", &call_credentials, &error_list_google_grpc)) { - std::vector<grpc_error*> error_list_call_credentials = + std::vector<grpc_error_handle> error_list_call_credentials = ParseJsonObjectCallCredentials(*call_credentials); if (!error_list_call_credentials.empty()) { error_list_google_grpc.push_back(GRPC_ERROR_CREATE_FROM_VECTOR( @@ -137,14 +137,14 @@ GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectGoogleGrpc( return error_list_google_grpc; } -std::vector<grpc_error*> +std::vector<grpc_error_handle> GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectGrpcServices( const Json::Object& grpc_service) { - std::vector<grpc_error*> error_list_grpc_services; + std::vector<grpc_error_handle> error_list_grpc_services; const Json::Object* google_grpc = nullptr; if (ParseJsonObjectField(grpc_service, "google_grpc", &google_grpc, &error_list_grpc_services)) { - std::vector<grpc_error*> error_list_google_grpc = + std::vector<grpc_error_handle> error_list_google_grpc = ParseJsonObjectGoogleGrpc(*google_grpc); if (!error_list_google_grpc.empty()) { error_list_grpc_services.push_back(GRPC_ERROR_CREATE_FROM_VECTOR( @@ -158,10 +158,10 @@ GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectGrpcServices( return error_list_grpc_services; } -std::vector<grpc_error*> +std::vector<grpc_error_handle> GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectServer( const Json::Object& server) { - std::vector<grpc_error*> error_list_server; + std::vector<grpc_error_handle> error_list_server; std::string api_type; if (ParseJsonObjectField(server, "api_type", &api_type, &error_list_server, false)) { @@ -180,7 +180,7 @@ GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectServer( const Json::Object* grpc_service = nullptr; if (ExtractJsonType((*grpc_services)[0], "grpc_services[0]", &grpc_service, &error_list_server)) { - std::vector<grpc_error*> error_list_grpc_services = + std::vector<grpc_error_handle> error_list_grpc_services = ParseJsonObjectGrpcServices(*grpc_service); if (!error_list_grpc_services.empty()) { error_list_server.push_back(GRPC_ERROR_CREATE_FROM_VECTOR( @@ -193,8 +193,8 @@ GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectServer( } RefCountedPtr<GoogleMeshCaCertificateProviderFactory::Config> -GoogleMeshCaCertificateProviderFactory::Config::Parse(const Json& config_json, - grpc_error** error) { +GoogleMeshCaCertificateProviderFactory::Config::Parse( + const Json& config_json, grpc_error_handle* error) { auto config = MakeRefCounted<GoogleMeshCaCertificateProviderFactory::Config>(); if (config_json.type() != Json::Type::OBJECT) { @@ -202,11 +202,11 @@ GoogleMeshCaCertificateProviderFactory::Config::Parse(const Json& config_json, "error:config type should be OBJECT."); return nullptr; } - std::vector<grpc_error*> error_list; + std::vector<grpc_error_handle> error_list; const Json::Object* server = nullptr; if (ParseJsonObjectField(config_json.object_value(), "server", &server, &error_list)) { - std::vector<grpc_error*> error_list_server = + std::vector<grpc_error_handle> error_list_server = config->ParseJsonObjectServer(*server); if (!error_list_server.empty()) { error_list.push_back( @@ -257,7 +257,7 @@ const char* GoogleMeshCaCertificateProviderFactory::name() const { RefCountedPtr<CertificateProviderFactory::Config> GoogleMeshCaCertificateProviderFactory::CreateCertificateProviderConfig( - const Json& config_json, grpc_error** error) { + const Json& config_json, grpc_error_handle* error) { return GoogleMeshCaCertificateProviderFactory::Config::Parse(config_json, error); } diff --git a/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.h b/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.h index f2765d6d..7a33f977 100644 --- a/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.h +++ b/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.h @@ -63,19 +63,20 @@ class GoogleMeshCaCertificateProviderFactory const std::string& location() const { return location_; } static RefCountedPtr<Config> Parse(const Json& config_json, - grpc_error** error); + grpc_error_handle* error); private: // Helpers for parsing the config - std::vector<grpc_error*> ParseJsonObjectStsService( + std::vector<grpc_error_handle> ParseJsonObjectStsService( const Json::Object& sts_service); - std::vector<grpc_error*> ParseJsonObjectCallCredentials( + std::vector<grpc_error_handle> ParseJsonObjectCallCredentials( const Json::Object& call_credentials); - std::vector<grpc_error*> ParseJsonObjectGoogleGrpc( + std::vector<grpc_error_handle> ParseJsonObjectGoogleGrpc( const Json::Object& google_grpc); - std::vector<grpc_error*> ParseJsonObjectGrpcServices( + std::vector<grpc_error_handle> ParseJsonObjectGrpcServices( const Json::Object& grpc_service); - std::vector<grpc_error*> ParseJsonObjectServer(const Json::Object& server); + std::vector<grpc_error_handle> ParseJsonObjectServer( + const Json::Object& server); std::string endpoint_; StsConfig sts_config_; @@ -90,10 +91,10 @@ class GoogleMeshCaCertificateProviderFactory RefCountedPtr<CertificateProviderFactory::Config> CreateCertificateProviderConfig(const Json& config_json, - grpc_error** error) override; + grpc_error_handle* error) override; RefCountedPtr<grpc_tls_certificate_provider> CreateCertificateProvider( - RefCountedPtr<CertificateProviderFactory::Config> config) override { + RefCountedPtr<CertificateProviderFactory::Config> /*config*/) override { // TODO(yashykt) : To be implemented return nullptr; } diff --git a/grpc/src/core/ext/xds/xds_api.cc b/grpc/src/core/ext/xds/xds_api.cc index e9403c2c..e51bc07c 100644 --- a/grpc/src/core/ext/xds/xds_api.cc +++ b/grpc/src/core/ext/xds/xds_api.cc @@ -28,26 +28,13 @@ #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" - -#include "upb/upb.hpp" - -#include <grpc/impl/codegen/log.h> -#include <grpc/support/alloc.h> -#include <grpc/support/string_util.h> - -#include "src/core/ext/xds/xds_api.h" -#include "src/core/lib/gpr/env.h" -#include "src/core/lib/gpr/string.h" -#include "src/core/lib/gpr/useful.h" -#include "src/core/lib/iomgr/error.h" -#include "src/core/lib/iomgr/sockaddr_utils.h" -#include "src/core/lib/slice/slice_utils.h" - +#include "envoy/admin/v3/config_dump.upb.h" #include "envoy/config/cluster/v3/circuit_breaker.upb.h" #include "envoy/config/cluster/v3/cluster.upb.h" #include "envoy/config/cluster/v3/cluster.upbdefs.h" #include "envoy/config/core/v3/address.upb.h" #include "envoy/config/core/v3/base.upb.h" +#include "envoy/config/core/v3/base.upbdefs.h" #include "envoy/config/core/v3/config_source.upb.h" #include "envoy/config/core/v3/health_check.upb.h" #include "envoy/config/core/v3/protocol.upb.h" @@ -57,12 +44,19 @@ #include "envoy/config/endpoint/v3/load_report.upb.h" #include "envoy/config/listener/v3/api_listener.upb.h" #include "envoy/config/listener/v3/listener.upb.h" +#include "envoy/config/listener/v3/listener.upbdefs.h" +#include "envoy/config/listener/v3/listener_components.upb.h" #include "envoy/config/route/v3/route.upb.h" #include "envoy/config/route/v3/route.upbdefs.h" #include "envoy/config/route/v3/route_components.upb.h" +#include "envoy/config/route/v3/route_components.upbdefs.h" +#include "envoy/extensions/clusters/aggregate/v3/cluster.upb.h" +#include "envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h" +#include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.h" #include "envoy/extensions/transport_sockets/tls/v3/common.upb.h" #include "envoy/extensions/transport_sockets/tls/v3/tls.upb.h" +#include "envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.h" #include "envoy/service/cluster/v3/cds.upb.h" #include "envoy/service/cluster/v3/cds.upbdefs.h" #include "envoy/service/discovery/v3/discovery.upb.h" @@ -74,6 +68,8 @@ #include "envoy/service/load_stats/v3/lrs.upbdefs.h" #include "envoy/service/route/v3/rds.upb.h" #include "envoy/service/route/v3/rds.upbdefs.h" +#include "envoy/service/status/v3/csds.upb.h" +#include "envoy/service/status/v3/csds.upbdefs.h" #include "envoy/type/matcher/v3/regex.upb.h" #include "envoy/type/matcher/v3/string.upb.h" #include "envoy/type/v3/percent.upb.h" @@ -81,18 +77,49 @@ #include "google/protobuf/any.upb.h" #include "google/protobuf/duration.upb.h" #include "google/protobuf/struct.upb.h" +#include "google/protobuf/timestamp.upb.h" #include "google/protobuf/wrappers.upb.h" #include "google/rpc/status.upb.h" +#include "udpa/type/v1/typed_struct.upb.h" #include "upb/text_encode.h" #include "upb/upb.h" +#include "upb/upb.hpp" + +#include <grpc/impl/codegen/log.h> +#include <grpc/support/alloc.h> +#include <grpc/support/string_util.h> + +#include "src/core/ext/xds/xds_api.h" +#include "src/core/lib/address_utils/sockaddr_utils.h" +#include "src/core/lib/gpr/env.h" +#include "src/core/lib/gpr/string.h" +#include "src/core/lib/gpr/useful.h" +#include "src/core/lib/gprpp/host_port.h" +#include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/sockaddr.h" +#include "src/core/lib/iomgr/socket_utils.h" +#include "src/core/lib/slice/slice_utils.h" namespace grpc_core { -// TODO (donnadionne): Check to see if timeout is enabled, this will be -// removed once timeout feature is fully integration-tested and enabled by +// TODO(donnadionne): Check to see if cluster types aggregate_cluster and +// logical_dns are enabled, this will be +// removed once the cluster types are fully integration-tested and enabled by +// default. +bool XdsAggregateAndLogicalDnsClusterEnabled() { + char* value = gpr_getenv( + "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER"); + bool parsed_value; + bool parse_succeeded = gpr_parse_bool_value(value, &parsed_value); + gpr_free(value); + return parse_succeeded && parsed_value; +} + +// TODO(donnadionne): Check to see if ring hash policy is enabled, this will be +// removed once ring hash policy is fully integration-tested and enabled by // default. -bool XdsTimeoutEnabled() { - char* value = gpr_getenv("GRPC_XDS_EXPERIMENTAL_ENABLE_TIMEOUT"); +bool XdsRingHashEnabled() { + char* value = gpr_getenv("GRPC_XDS_EXPERIMENTAL_ENABLE_RING_HASH"); bool parsed_value; bool parse_succeeded = gpr_parse_bool_value(value, &parsed_value); gpr_free(value); @@ -111,160 +138,80 @@ bool XdsSecurityEnabled() { } // -// XdsApi::Route::Matchers::PathMatcher +// XdsApi::Route::HashPolicy // -XdsApi::Route::Matchers::PathMatcher::PathMatcher(const PathMatcher& other) - : type(other.type), case_sensitive(other.case_sensitive) { - if (type == PathMatcherType::REGEX) { - RE2::Options options; - options.set_case_sensitive(case_sensitive); - regex_matcher = - absl::make_unique<RE2>(other.regex_matcher->pattern(), options); - } else { - string_matcher = other.string_matcher; +XdsApi::Route::HashPolicy::HashPolicy(const HashPolicy& other) + : type(other.type), + header_name(other.header_name), + regex_substitution(other.regex_substitution) { + if (other.regex != nullptr) { + regex = + absl::make_unique<RE2>(other.regex->pattern(), other.regex->options()); } } -XdsApi::Route::Matchers::PathMatcher& XdsApi::Route::Matchers::PathMatcher:: -operator=(const PathMatcher& other) { +XdsApi::Route::HashPolicy& XdsApi::Route::HashPolicy::operator=( + const HashPolicy& other) { type = other.type; - case_sensitive = other.case_sensitive; - if (type == PathMatcherType::REGEX) { - RE2::Options options; - options.set_case_sensitive(case_sensitive); - regex_matcher = - absl::make_unique<RE2>(other.regex_matcher->pattern(), options); - } else { - string_matcher = other.string_matcher; + header_name = other.header_name; + if (other.regex != nullptr) { + regex = + absl::make_unique<RE2>(other.regex->pattern(), other.regex->options()); } + regex_substitution = other.regex_substitution; return *this; } -bool XdsApi::Route::Matchers::PathMatcher::operator==( - const PathMatcher& other) const { - if (type != other.type) return false; - if (case_sensitive != other.case_sensitive) return false; - if (type == PathMatcherType::REGEX) { - // Should never be null. - if (regex_matcher == nullptr || other.regex_matcher == nullptr) { - return false; - } - return regex_matcher->pattern() == other.regex_matcher->pattern(); - } - return string_matcher == other.string_matcher; -} - -std::string XdsApi::Route::Matchers::PathMatcher::ToString() const { - std::string path_type_string; - switch (type) { - case PathMatcherType::PATH: - path_type_string = "path match"; - break; - case PathMatcherType::PREFIX: - path_type_string = "prefix match"; - break; - case PathMatcherType::REGEX: - path_type_string = "regex match"; - break; - default: - break; - } - return absl::StrFormat("Path %s:%s%s", path_type_string, - type == PathMatcherType::REGEX - ? regex_matcher->pattern() - : string_matcher, - case_sensitive ? "" : "[case_sensitive=false]"); -} - -// -// XdsApi::Route::Matchers::HeaderMatcher -// - -XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcher( - const HeaderMatcher& other) - : name(other.name), type(other.type), invert_match(other.invert_match) { - switch (type) { - case HeaderMatcherType::REGEX: - regex_match = absl::make_unique<RE2>(other.regex_match->pattern()); - break; - case HeaderMatcherType::RANGE: - range_start = other.range_start; - range_end = other.range_end; - break; - case HeaderMatcherType::PRESENT: - present_match = other.present_match; - break; - default: - string_matcher = other.string_matcher; - } -} +XdsApi::Route::HashPolicy::HashPolicy(HashPolicy&& other) noexcept + : type(other.type), + header_name(std::move(other.header_name)), + regex(std::move(other.regex)), + regex_substitution(std::move(other.regex_substitution)) {} -XdsApi::Route::Matchers::HeaderMatcher& XdsApi::Route::Matchers::HeaderMatcher:: -operator=(const HeaderMatcher& other) { - name = other.name; +XdsApi::Route::HashPolicy& XdsApi::Route::HashPolicy::operator=( + HashPolicy&& other) noexcept { type = other.type; - invert_match = other.invert_match; - switch (type) { - case HeaderMatcherType::REGEX: - regex_match = absl::make_unique<RE2>(other.regex_match->pattern()); - break; - case HeaderMatcherType::RANGE: - range_start = other.range_start; - range_end = other.range_end; - break; - case HeaderMatcherType::PRESENT: - present_match = other.present_match; - break; - default: - string_matcher = other.string_matcher; - } + header_name = std::move(other.header_name); + regex = std::move(other.regex); + regex_substitution = std::move(other.regex_substitution); return *this; } -bool XdsApi::Route::Matchers::HeaderMatcher::operator==( - const HeaderMatcher& other) const { - if (name != other.name) return false; +bool XdsApi::Route::HashPolicy::HashPolicy::operator==( + const HashPolicy& other) const { if (type != other.type) return false; - if (invert_match != other.invert_match) return false; - switch (type) { - case HeaderMatcherType::REGEX: - return regex_match->pattern() != other.regex_match->pattern(); - case HeaderMatcherType::RANGE: - return range_start != other.range_start && range_end != other.range_end; - case HeaderMatcherType::PRESENT: - return present_match != other.present_match; - default: - return string_matcher != other.string_matcher; + if (type == Type::HEADER) { + if (regex == nullptr) { + if (other.regex != nullptr) return false; + } else { + if (other.regex == nullptr) return false; + return header_name == other.header_name && + regex->pattern() == other.regex->pattern() && + regex_substitution == other.regex_substitution; + } } + return true; } -std::string XdsApi::Route::Matchers::HeaderMatcher::ToString() const { +std::string XdsApi::Route::HashPolicy::ToString() const { + std::vector<std::string> contents; switch (type) { - case HeaderMatcherType::EXACT: - return absl::StrFormat("Header exact match:%s %s:%s", - invert_match ? " not" : "", name, string_matcher); - case HeaderMatcherType::REGEX: - return absl::StrFormat("Header regex match:%s %s:%s", - invert_match ? " not" : "", name, - regex_match->pattern()); - case HeaderMatcherType::RANGE: - return absl::StrFormat("Header range match:%s %s:[%d, %d)", - invert_match ? " not" : "", name, range_start, - range_end); - case HeaderMatcherType::PRESENT: - return absl::StrFormat("Header present match:%s %s:%s", - invert_match ? " not" : "", name, - present_match ? "true" : "false"); - case HeaderMatcherType::PREFIX: - return absl::StrFormat("Header prefix match:%s %s:%s", - invert_match ? " not" : "", name, string_matcher); - case HeaderMatcherType::SUFFIX: - return absl::StrFormat("Header suffix match:%s %s:%s", - invert_match ? " not" : "", name, string_matcher); - default: - return ""; + case Type::HEADER: + contents.push_back("type=HEADER"); + break; + case Type::CHANNEL_ID: + contents.push_back("type=CHANNEL_ID"); + break; + } + contents.push_back( + absl::StrFormat("terminal=%s", terminal ? "true" : "false")); + if (type == Type::HEADER) { + contents.push_back(absl::StrFormat( + "Header %s:/%s/%s", header_name, + (regex == nullptr) ? "" : regex->pattern(), regex_substitution)); } + return absl::StrCat("{", absl::StrJoin(contents, ", "), "}"); } // @@ -273,7 +220,8 @@ std::string XdsApi::Route::Matchers::HeaderMatcher::ToString() const { std::string XdsApi::Route::Matchers::ToString() const { std::vector<std::string> contents; - contents.push_back(path_matcher.ToString()); + contents.push_back( + absl::StrFormat("PathMatcher{%s}", path_matcher.ToString())); for (const HeaderMatcher& header_matcher : header_matchers) { contents.push_back(header_matcher.ToString()); } @@ -285,12 +233,28 @@ std::string XdsApi::Route::Matchers::ToString() const { } std::string XdsApi::Route::ClusterWeight::ToString() const { - return absl::StrFormat("{cluster=%s, weight=%d}", name, weight); + std::vector<std::string> contents; + contents.push_back(absl::StrCat("cluster=", name)); + contents.push_back(absl::StrCat("weight=", weight)); + if (!typed_per_filter_config.empty()) { + std::vector<std::string> parts; + for (const auto& p : typed_per_filter_config) { + const std::string& key = p.first; + const auto& config = p.second; + parts.push_back(absl::StrCat(key, "=", config.ToString())); + } + contents.push_back(absl::StrCat("typed_per_filter_config={", + absl::StrJoin(parts, ", "), "}")); + } + return absl::StrCat("{", absl::StrJoin(contents, ", "), "}"); } std::string XdsApi::Route::ToString() const { std::vector<std::string> contents; contents.push_back(matchers.ToString()); + for (const HashPolicy& hash_policy : hash_policies) { + contents.push_back(absl::StrCat("hash_policy=", hash_policy.ToString())); + } if (!cluster_name.empty()) { contents.push_back(absl::StrFormat("Cluster name: %s", cluster_name)); } @@ -300,6 +264,15 @@ std::string XdsApi::Route::ToString() const { if (max_stream_duration.has_value()) { contents.push_back(max_stream_duration->ToString()); } + if (!typed_per_filter_config.empty()) { + contents.push_back("typed_per_filter_config={"); + for (const auto& p : typed_per_filter_config) { + const std::string& name = p.first; + const auto& config = p.second; + contents.push_back(absl::StrCat(" ", name, "=", config.ToString())); + } + contents.push_back("}"); + } return absl::StrJoin(contents, "\n"); } @@ -322,6 +295,14 @@ std::string XdsApi::RdsUpdate::ToString() const { vhosts.push_back("\n }\n"); } vhosts.push_back(" ]\n"); + vhosts.push_back(" typed_per_filter_config={\n"); + for (const auto& p : vhost.typed_per_filter_config) { + const std::string& name = p.first; + const auto& config = p.second; + vhosts.push_back( + absl::StrCat(" ", name, "=", config.ToString(), "\n")); + } + vhosts.push_back(" }\n"); vhosts.push_back("]\n"); } return absl::StrJoin(vhosts, ""); @@ -426,102 +407,6 @@ XdsApi::RdsUpdate::VirtualHost* XdsApi::RdsUpdate::FindVirtualHostForDomain( } // -// XdsApi::StringMatcher -// - -XdsApi::StringMatcher::StringMatcher(StringMatcherType type, - const std::string& matcher, - bool ignore_case) - : type_(type), ignore_case_(ignore_case) { - if (type_ == StringMatcherType::SAFE_REGEX) { - regex_matcher_ = absl::make_unique<RE2>(matcher); - } else { - string_matcher_ = matcher; - } -} - -XdsApi::StringMatcher::StringMatcher(const StringMatcher& other) - : type_(other.type_), ignore_case_(other.ignore_case_) { - switch (type_) { - case StringMatcherType::SAFE_REGEX: - regex_matcher_ = absl::make_unique<RE2>(other.regex_matcher_->pattern()); - break; - default: - string_matcher_ = other.string_matcher_; - } -} - -XdsApi::StringMatcher& XdsApi::StringMatcher::operator=( - const StringMatcher& other) { - type_ = other.type_; - switch (type_) { - case StringMatcherType::SAFE_REGEX: - regex_matcher_ = absl::make_unique<RE2>(other.regex_matcher_->pattern()); - break; - default: - string_matcher_ = other.string_matcher_; - } - ignore_case_ = other.ignore_case_; - return *this; -} - -bool XdsApi::StringMatcher::operator==(const StringMatcher& other) const { - if (type_ != other.type_ || ignore_case_ != other.ignore_case_) return false; - switch (type_) { - case StringMatcherType::SAFE_REGEX: - return regex_matcher_->pattern() == other.regex_matcher_->pattern(); - default: - return string_matcher_ == other.string_matcher_; - } -} - -bool XdsApi::StringMatcher::Match(absl::string_view value) const { - switch (type_) { - case XdsApi::StringMatcher::StringMatcherType::EXACT: - return ignore_case_ ? absl::EqualsIgnoreCase(value, string_matcher_) - : value == string_matcher_; - case XdsApi::StringMatcher::StringMatcherType::PREFIX: - return ignore_case_ ? absl::StartsWithIgnoreCase(value, string_matcher_) - : absl::StartsWith(value, string_matcher_); - case XdsApi::StringMatcher::StringMatcherType::SUFFIX: - return ignore_case_ ? absl::EndsWithIgnoreCase(value, string_matcher_) - : absl::EndsWith(value, string_matcher_); - case XdsApi::StringMatcher::StringMatcherType::CONTAINS: - return ignore_case_ - ? absl::StrContains(absl::AsciiStrToLower(value), - absl::AsciiStrToLower(string_matcher_)) - : absl::StrContains(value, string_matcher_); - case XdsApi::StringMatcher::StringMatcherType::SAFE_REGEX: - // ignore_case_ is ignored for SAFE_REGEX - return RE2::FullMatch(std::string(value), *regex_matcher_); - default: - return false; - } -} - -std::string XdsApi::StringMatcher::ToString() const { - switch (type_) { - case StringMatcherType::EXACT: - return absl::StrFormat("StringMatcher{exact=%s%s}", string_matcher_, - ignore_case_ ? ", ignore_case" : ""); - case StringMatcherType::PREFIX: - return absl::StrFormat("StringMatcher{prefix=%s%s}", string_matcher_, - ignore_case_ ? ", ignore_case" : ""); - case StringMatcherType::SUFFIX: - return absl::StrFormat("StringMatcher{suffix=%s%s}", string_matcher_, - ignore_case_ ? ", ignore_case" : ""); - case StringMatcherType::CONTAINS: - return absl::StrFormat("StringMatcher{contains=%s%s}", string_matcher_, - ignore_case_ ? ", ignore_case" : ""); - case StringMatcherType::SAFE_REGEX: - return absl::StrFormat("StringMatcher{safe_regex=%s}", - regex_matcher_->pattern()); - default: - return ""; - } -} - -// // XdsApi::CommonTlsContext::CertificateValidationContext // @@ -610,6 +495,204 @@ bool XdsApi::CommonTlsContext::Empty() const { } // +// XdsApi::DownstreamTlsContext +// + +std::string XdsApi::DownstreamTlsContext::ToString() const { + return absl::StrFormat("common_tls_context=%s, require_client_certificate=%s", + common_tls_context.ToString(), + require_client_certificate ? "true" : "false"); +} + +bool XdsApi::DownstreamTlsContext::Empty() const { + return common_tls_context.Empty(); +} + +// +// XdsApi::LdsUpdate::HttpConnectionManager +// + +std::string XdsApi::LdsUpdate::HttpConnectionManager::ToString() const { + absl::InlinedVector<std::string, 4> contents; + contents.push_back(absl::StrFormat( + "route_config_name=%s", + !route_config_name.empty() ? route_config_name.c_str() : "<inlined>")); + contents.push_back(absl::StrFormat("http_max_stream_duration=%s", + http_max_stream_duration.ToString())); + if (rds_update.has_value()) { + contents.push_back( + absl::StrFormat("rds_update=%s", rds_update->ToString())); + } + if (!http_filters.empty()) { + std::vector<std::string> filter_strings; + for (const auto& http_filter : http_filters) { + filter_strings.push_back(http_filter.ToString()); + } + contents.push_back(absl::StrCat("http_filters=[", + absl::StrJoin(filter_strings, ", "), "]")); + } + return absl::StrCat("{", absl::StrJoin(contents, ", "), "}"); +} + +// +// XdsApi::LdsUpdate::HttpFilter +// + +std::string XdsApi::LdsUpdate::HttpConnectionManager::HttpFilter::ToString() + const { + return absl::StrCat("{name=", name, ", config=", config.ToString(), "}"); +} + +// +// XdsApi::LdsUpdate::FilterChainData +// + +std::string XdsApi::LdsUpdate::FilterChainData::ToString() const { + return absl::StrCat( + "{downstream_tls_context=", downstream_tls_context.ToString(), + " http_connection_manager=", http_connection_manager.ToString(), "}"); +} + +// +// XdsApi::LdsUpdate::FilterChainMap::CidrRange +// + +std::string XdsApi::LdsUpdate::FilterChainMap::CidrRange::ToString() const { + return absl::StrCat( + "{address_prefix=", grpc_sockaddr_to_string(&address, false), + ", prefix_len=", prefix_len, "}"); +} + +// +// FilterChain +// + +struct FilterChain { + struct FilterChainMatch { + uint32_t destination_port = 0; + std::vector<XdsApi::LdsUpdate::FilterChainMap::CidrRange> prefix_ranges; + XdsApi::LdsUpdate::FilterChainMap::ConnectionSourceType source_type = + XdsApi::LdsUpdate::FilterChainMap::ConnectionSourceType::kAny; + std::vector<XdsApi::LdsUpdate::FilterChainMap::CidrRange> + source_prefix_ranges; + std::vector<uint32_t> source_ports; + std::vector<std::string> server_names; + std::string transport_protocol; + std::vector<std::string> application_protocols; + + std::string ToString() const; + } filter_chain_match; + + std::shared_ptr<XdsApi::LdsUpdate::FilterChainData> filter_chain_data; +}; + +std::string FilterChain::FilterChainMatch::ToString() const { + absl::InlinedVector<std::string, 8> contents; + if (destination_port != 0) { + contents.push_back(absl::StrCat("destination_port=", destination_port)); + } + if (!prefix_ranges.empty()) { + std::vector<std::string> prefix_ranges_content; + for (const auto& range : prefix_ranges) { + prefix_ranges_content.push_back(range.ToString()); + } + contents.push_back(absl::StrCat( + "prefix_ranges={", absl::StrJoin(prefix_ranges_content, ", "), "}")); + } + if (source_type == XdsApi::LdsUpdate::FilterChainMap::ConnectionSourceType:: + kSameIpOrLoopback) { + contents.push_back("source_type=SAME_IP_OR_LOOPBACK"); + } else if (source_type == XdsApi::LdsUpdate::FilterChainMap:: + ConnectionSourceType::kExternal) { + contents.push_back("source_type=EXTERNAL"); + } + if (!source_prefix_ranges.empty()) { + std::vector<std::string> source_prefix_ranges_content; + for (const auto& range : source_prefix_ranges) { + source_prefix_ranges_content.push_back(range.ToString()); + } + contents.push_back( + absl::StrCat("source_prefix_ranges={", + absl::StrJoin(source_prefix_ranges_content, ", "), "}")); + } + if (!source_ports.empty()) { + contents.push_back( + absl::StrCat("source_ports={", absl::StrJoin(source_ports, ", "), "}")); + } + if (!server_names.empty()) { + contents.push_back( + absl::StrCat("server_names={", absl::StrJoin(server_names, ", "), "}")); + } + if (!transport_protocol.empty()) { + contents.push_back(absl::StrCat("transport_protocol=", transport_protocol)); + } + if (!application_protocols.empty()) { + contents.push_back(absl::StrCat("application_protocols={", + absl::StrJoin(application_protocols, ", "), + "}")); + } + return absl::StrCat("{", absl::StrJoin(contents, ", "), "}"); +} + +// +// XdsApi::LdsUpdate::FilterChainMap +// + +std::string XdsApi::LdsUpdate::FilterChainMap::ToString() const { + std::vector<std::string> contents; + for (const auto& destination_ip : destination_ip_vector) { + for (int source_type = 0; source_type < 3; ++source_type) { + for (const auto& source_ip : + destination_ip.source_types_array[source_type]) { + for (const auto& source_port_pair : source_ip.ports_map) { + FilterChain::FilterChainMatch filter_chain_match; + if (destination_ip.prefix_range.has_value()) { + filter_chain_match.prefix_ranges.push_back( + *destination_ip.prefix_range); + } + filter_chain_match.source_type = static_cast< + XdsApi::LdsUpdate::FilterChainMap::ConnectionSourceType>( + source_type); + if (source_ip.prefix_range.has_value()) { + filter_chain_match.source_prefix_ranges.push_back( + *source_ip.prefix_range); + } + if (source_port_pair.first != 0) { + filter_chain_match.source_ports.push_back(source_port_pair.first); + } + contents.push_back(absl::StrCat( + "{filter_chain_match=", filter_chain_match.ToString(), + ", filter_chain=", source_port_pair.second.data->ToString(), + "}")); + } + } + } + } + return absl::StrCat("{", absl::StrJoin(contents, ", "), "}"); +} + +// +// XdsApi::LdsUpdate +// + +std::string XdsApi::LdsUpdate::ToString() const { + absl::InlinedVector<std::string, 4> contents; + if (type == ListenerType::kTcpListener) { + contents.push_back(absl::StrCat("address=", address)); + contents.push_back( + absl::StrCat("filter_chain_map=", filter_chain_map.ToString())); + if (default_filter_chain.has_value()) { + contents.push_back(absl::StrCat("default_filter_chain=", + default_filter_chain->ToString())); + } + } else if (type == ListenerType::kHttpApiListener) { + contents.push_back(absl::StrFormat("http_connection_manager=%s", + http_connection_manager.ToString())); + } + return absl::StrCat("{", absl::StrJoin(contents, ", "), "}"); +} + +// // XdsApi::CdsUpdate // @@ -724,8 +807,13 @@ const char* kCdsV2TypeUrl = "type.googleapis.com/envoy.api.v2.Cluster"; const char* kEdsV2TypeUrl = "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"; -bool IsLds(absl::string_view type_url) { - return type_url == XdsApi::kLdsTypeUrl || type_url == kLdsV2TypeUrl; +bool IsLds(absl::string_view type_url, bool* is_v2 = nullptr) { + if (type_url == XdsApi::kLdsTypeUrl) return true; + if (type_url == kLdsV2TypeUrl) { + if (is_v2 != nullptr) *is_v2 = true; + return true; + } + return false; } bool IsRds(absl::string_view type_url) { @@ -749,39 +837,67 @@ XdsApi::XdsApi(XdsClient* client, TraceFlag* tracer, node_(node), build_version_(absl::StrCat("gRPC C-core ", GPR_PLATFORM_STRING, " ", grpc_version_string())), - user_agent_name_(absl::StrCat("gRPC C-core ", GPR_PLATFORM_STRING)) {} + user_agent_name_(absl::StrCat("gRPC C-core ", GPR_PLATFORM_STRING)) { + // Populate upb symtab with xDS proto messages that we want to print + // properly in logs. + // Note: This won't actually work properly until upb adds support for + // Any fields in textproto printing (internal b/178821188). + envoy_config_listener_v3_Listener_getmsgdef(symtab_.ptr()); + envoy_config_route_v3_RouteConfiguration_getmsgdef(symtab_.ptr()); + envoy_config_cluster_v3_Cluster_getmsgdef(symtab_.ptr()); + envoy_extensions_clusters_aggregate_v3_ClusterConfig_getmsgdef(symtab_.ptr()); + envoy_config_cluster_v3_Cluster_getmsgdef(symtab_.ptr()); + envoy_config_endpoint_v3_ClusterLoadAssignment_getmsgdef(symtab_.ptr()); + envoy_extensions_transport_sockets_tls_v3_UpstreamTlsContext_getmsgdef( + symtab_.ptr()); + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_getmsgdef( + symtab_.ptr()); + // Load HTTP filter proto messages into the upb symtab. + XdsHttpFilterRegistry::PopulateSymtab(symtab_.ptr()); +} namespace { +struct EncodingContext { + XdsClient* client; + TraceFlag* tracer; + upb_symtab* symtab; + upb_arena* arena; + bool use_v3; +}; + // Works for both std::string and absl::string_view. template <typename T> inline upb_strview StdStringToUpbString(const T& str) { return upb_strview_make(str.data(), str.size()); } -void PopulateMetadataValue(upb_arena* arena, google_protobuf_Value* value_pb, - const Json& value); +void PopulateMetadataValue(const EncodingContext& context, + google_protobuf_Value* value_pb, const Json& value); -void PopulateListValue(upb_arena* arena, google_protobuf_ListValue* list_value, +void PopulateListValue(const EncodingContext& context, + google_protobuf_ListValue* list_value, const Json::Array& values) { for (const auto& value : values) { - auto* value_pb = google_protobuf_ListValue_add_values(list_value, arena); - PopulateMetadataValue(arena, value_pb, value); + auto* value_pb = + google_protobuf_ListValue_add_values(list_value, context.arena); + PopulateMetadataValue(context, value_pb, value); } } -void PopulateMetadata(upb_arena* arena, google_protobuf_Struct* metadata_pb, +void PopulateMetadata(const EncodingContext& context, + google_protobuf_Struct* metadata_pb, const Json::Object& metadata) { for (const auto& p : metadata) { - google_protobuf_Value* value = google_protobuf_Value_new(arena); - PopulateMetadataValue(arena, value, p.second); + google_protobuf_Value* value = google_protobuf_Value_new(context.arena); + PopulateMetadataValue(context, value, p.second); google_protobuf_Struct_fields_set( - metadata_pb, StdStringToUpbString(p.first), value, arena); + metadata_pb, StdStringToUpbString(p.first), value, context.arena); } } -void PopulateMetadataValue(upb_arena* arena, google_protobuf_Value* value_pb, - const Json& value) { +void PopulateMetadataValue(const EncodingContext& context, + google_protobuf_Value* value_pb, const Json& value) { switch (value.type()) { case Json::Type::JSON_NULL: google_protobuf_Value_set_null_value(value_pb, 0); @@ -802,14 +918,14 @@ void PopulateMetadataValue(upb_arena* arena, google_protobuf_Value* value_pb, break; case Json::Type::OBJECT: { google_protobuf_Struct* struct_value = - google_protobuf_Value_mutable_struct_value(value_pb, arena); - PopulateMetadata(arena, struct_value, value.object_value()); + google_protobuf_Value_mutable_struct_value(value_pb, context.arena); + PopulateMetadata(context, struct_value, value.object_value()); break; } case Json::Type::ARRAY: { google_protobuf_ListValue* list_value = - google_protobuf_Value_mutable_list_value(value_pb, arena); - PopulateListValue(arena, list_value, value.array_value()); + google_protobuf_Value_mutable_list_value(value_pb, context.arena); + PopulateListValue(context, list_value, value.array_value()); break; } } @@ -836,7 +952,8 @@ std::string EncodeStringField(uint32_t field_number, const std::string& str) { EncodeVarint(str.size()) + str; } -void PopulateBuildVersion(upb_arena* arena, envoy_config_core_v3_Node* node_msg, +void PopulateBuildVersion(const EncodingContext& context, + envoy_config_core_v3_Node* node_msg, const std::string& build_version) { std::string encoded_build_version = EncodeStringField(5, build_version); // TODO(roth): This should use upb_msg_addunknown(), but that API is @@ -844,10 +961,11 @@ void PopulateBuildVersion(upb_arena* arena, envoy_config_core_v3_Node* node_msg, // API for now. Change this once we upgrade to a version of upb that // fixes this bug. _upb_msg_addunknown(node_msg, encoded_build_version.data(), - encoded_build_version.size(), arena); + encoded_build_version.size(), context.arena); } -void PopulateNode(upb_arena* arena, const XdsBootstrap::Node* node, bool use_v3, +void PopulateNode(const EncodingContext& context, + const XdsBootstrap::Node* node, const std::string& build_version, const std::string& user_agent_name, envoy_config_core_v3_Node* node_msg) { @@ -862,13 +980,13 @@ void PopulateNode(upb_arena* arena, const XdsBootstrap::Node* node, bool use_v3, } if (!node->metadata.object_value().empty()) { google_protobuf_Struct* metadata = - envoy_config_core_v3_Node_mutable_metadata(node_msg, arena); - PopulateMetadata(arena, metadata, node->metadata.object_value()); + envoy_config_core_v3_Node_mutable_metadata(node_msg, context.arena); + PopulateMetadata(context, metadata, node->metadata.object_value()); } if (!node->locality_region.empty() || !node->locality_zone.empty() || - !node->locality_subzone.empty()) { + !node->locality_sub_zone.empty()) { envoy_config_core_v3_Locality* locality = - envoy_config_core_v3_Node_mutable_locality(node_msg, arena); + envoy_config_core_v3_Node_mutable_locality(node_msg, context.arena); if (!node->locality_region.empty()) { envoy_config_core_v3_Locality_set_region( locality, StdStringToUpbString(node->locality_region)); @@ -877,14 +995,14 @@ void PopulateNode(upb_arena* arena, const XdsBootstrap::Node* node, bool use_v3, envoy_config_core_v3_Locality_set_zone( locality, StdStringToUpbString(node->locality_zone)); } - if (!node->locality_subzone.empty()) { + if (!node->locality_sub_zone.empty()) { envoy_config_core_v3_Locality_set_sub_zone( - locality, StdStringToUpbString(node->locality_subzone)); + locality, StdStringToUpbString(node->locality_sub_zone)); } } } - if (!use_v3) { - PopulateBuildVersion(arena, node_msg, build_version); + if (!context.use_v3) { + PopulateBuildVersion(context, node_msg, build_version); } envoy_config_core_v3_Node_set_user_agent_name( node_msg, StdStringToUpbString(user_agent_name)); @@ -892,7 +1010,7 @@ void PopulateNode(upb_arena* arena, const XdsBootstrap::Node* node, bool use_v3, node_msg, upb_strview_makez(grpc_version_string())); envoy_config_core_v3_Node_add_client_features( node_msg, upb_strview_makez("envoy.lb.does_not_support_overprovisioning"), - arena); + context.arena); } inline absl::string_view UpbStringToAbsl(const upb_strview& str) { @@ -904,24 +1022,25 @@ inline std::string UpbStringToStdString(const upb_strview& str) { } void MaybeLogDiscoveryRequest( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, + const EncodingContext& context, const envoy_service_discovery_v3_DiscoveryRequest* request) { - if (GRPC_TRACE_FLAG_ENABLED(*tracer) && + if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { const upb_msgdef* msg_type = - envoy_service_discovery_v3_DiscoveryRequest_getmsgdef(symtab); + envoy_service_discovery_v3_DiscoveryRequest_getmsgdef(context.symtab); char buf[10240]; upb_text_encode(request, msg_type, nullptr, 0, buf, sizeof(buf)); - gpr_log(GPR_DEBUG, "[xds_client %p] constructed ADS request: %s", client, - buf); + gpr_log(GPR_DEBUG, "[xds_client %p] constructed ADS request: %s", + context.client, buf); } } grpc_slice SerializeDiscoveryRequest( - upb_arena* arena, envoy_service_discovery_v3_DiscoveryRequest* request) { + const EncodingContext& context, + envoy_service_discovery_v3_DiscoveryRequest* request) { size_t output_length; char* output = envoy_service_discovery_v3_DiscoveryRequest_serialize( - request, arena, &output_length); + request, context.arena, &output_length); return grpc_slice_from_copied_buffer(output, output_length); } @@ -949,9 +1068,11 @@ absl::string_view TypeUrlExternalToInternal(bool use_v3, grpc_slice XdsApi::CreateAdsRequest( const XdsBootstrap::XdsServer& server, const std::string& type_url, const std::set<absl::string_view>& resource_names, - const std::string& version, const std::string& nonce, grpc_error* error, - bool populate_node) { + const std::string& version, const std::string& nonce, + grpc_error_handle error, bool populate_node) { upb::Arena arena; + const EncodingContext context = {client_, tracer_, symtab_.ptr(), arena.ptr(), + server.ShouldUseV3()}; // Create a request. envoy_service_discovery_v3_DiscoveryRequest* request = envoy_service_discovery_v3_DiscoveryRequest_new(arena.ptr()); @@ -971,6 +1092,7 @@ grpc_slice XdsApi::CreateAdsRequest( request, StdStringToUpbString(nonce)); } // Set error_detail if it's a NACK. + std::string error_string_storage; if (error != GRPC_ERROR_NONE) { google_rpc_Status* error_detail = envoy_service_discovery_v3_DiscoveryRequest_mutable_error_detail( @@ -981,12 +1103,9 @@ grpc_slice XdsApi::CreateAdsRequest( // generate them in the parsing code, and then use that here. google_rpc_Status_set_code(error_detail, GRPC_STATUS_INVALID_ARGUMENT); // Error description comes from the error that was passed in. - grpc_slice error_description_slice; - GPR_ASSERT(grpc_error_get_str(error, GRPC_ERROR_STR_DESCRIPTION, - &error_description_slice)); - upb_strview error_description_strview = - StdStringToUpbString(StringViewFromSlice(error_description_slice)); - google_rpc_Status_set_message(error_detail, error_description_strview); + error_string_storage = grpc_error_std_string(error); + upb_strview error_description = StdStringToUpbString(error_string_storage); + google_rpc_Status_set_message(error_detail, error_description); GRPC_ERROR_UNREF(error); } // Populate node. @@ -994,79 +1113,102 @@ grpc_slice XdsApi::CreateAdsRequest( envoy_config_core_v3_Node* node_msg = envoy_service_discovery_v3_DiscoveryRequest_mutable_node(request, arena.ptr()); - PopulateNode(arena.ptr(), node_, server.ShouldUseV3(), build_version_, - user_agent_name_, node_msg); + PopulateNode(context, node_, build_version_, user_agent_name_, node_msg); } // Add resource_names. for (const auto& resource_name : resource_names) { envoy_service_discovery_v3_DiscoveryRequest_add_resource_names( request, StdStringToUpbString(resource_name), arena.ptr()); } - MaybeLogDiscoveryRequest(client_, tracer_, symtab_.ptr(), request); - return SerializeDiscoveryRequest(arena.ptr(), request); + MaybeLogDiscoveryRequest(context, request); + return SerializeDiscoveryRequest(context, request); } namespace { void MaybeLogDiscoveryResponse( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, + const EncodingContext& context, const envoy_service_discovery_v3_DiscoveryResponse* response) { - if (GRPC_TRACE_FLAG_ENABLED(*tracer) && + if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { const upb_msgdef* msg_type = - envoy_service_discovery_v3_DiscoveryResponse_getmsgdef(symtab); + envoy_service_discovery_v3_DiscoveryResponse_getmsgdef(context.symtab); char buf[10240]; upb_text_encode(response, msg_type, nullptr, 0, buf, sizeof(buf)); - gpr_log(GPR_DEBUG, "[xds_client %p] received response: %s", client, buf); + gpr_log(GPR_DEBUG, "[xds_client %p] received response: %s", context.client, + buf); + } +} + +void MaybeLogHttpConnectionManager( + const EncodingContext& context, + const envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager* + http_connection_manager_config) { + if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && + gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { + const upb_msgdef* msg_type = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_getmsgdef( + context.symtab); + char buf[10240]; + upb_text_encode(http_connection_manager_config, msg_type, nullptr, 0, buf, + sizeof(buf)); + gpr_log(GPR_DEBUG, "[xds_client %p] HttpConnectionManager: %s", + context.client, buf); } } void MaybeLogRouteConfiguration( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, + const EncodingContext& context, const envoy_config_route_v3_RouteConfiguration* route_config) { - if (GRPC_TRACE_FLAG_ENABLED(*tracer) && + if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { const upb_msgdef* msg_type = - envoy_config_route_v3_RouteConfiguration_getmsgdef(symtab); + envoy_config_route_v3_RouteConfiguration_getmsgdef(context.symtab); char buf[10240]; upb_text_encode(route_config, msg_type, nullptr, 0, buf, sizeof(buf)); - gpr_log(GPR_DEBUG, "[xds_client %p] RouteConfiguration: %s", client, buf); + gpr_log(GPR_DEBUG, "[xds_client %p] RouteConfiguration: %s", context.client, + buf); } } -void MaybeLogCluster(XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, +void MaybeLogCluster(const EncodingContext& context, const envoy_config_cluster_v3_Cluster* cluster) { - if (GRPC_TRACE_FLAG_ENABLED(*tracer) && + if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { const upb_msgdef* msg_type = - envoy_config_cluster_v3_Cluster_getmsgdef(symtab); + envoy_config_cluster_v3_Cluster_getmsgdef(context.symtab); char buf[10240]; upb_text_encode(cluster, msg_type, nullptr, 0, buf, sizeof(buf)); - gpr_log(GPR_DEBUG, "[xds_client %p] Cluster: %s", client, buf); + gpr_log(GPR_DEBUG, "[xds_client %p] Cluster: %s", context.client, buf); } } void MaybeLogClusterLoadAssignment( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, + const EncodingContext& context, const envoy_config_endpoint_v3_ClusterLoadAssignment* cla) { - if (GRPC_TRACE_FLAG_ENABLED(*tracer) && + if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { const upb_msgdef* msg_type = - envoy_config_endpoint_v3_ClusterLoadAssignment_getmsgdef(symtab); + envoy_config_endpoint_v3_ClusterLoadAssignment_getmsgdef( + context.symtab); char buf[10240]; upb_text_encode(cla, msg_type, nullptr, 0, buf, sizeof(buf)); - gpr_log(GPR_DEBUG, "[xds_client %p] ClusterLoadAssignment: %s", client, - buf); + gpr_log(GPR_DEBUG, "[xds_client %p] ClusterLoadAssignment: %s", + context.client, buf); } } -grpc_error* RoutePathMatchParse(const envoy_config_route_v3_RouteMatch* match, - XdsApi::Route* route, bool* ignore_route) { - auto* case_sensitive = envoy_config_route_v3_RouteMatch_case_sensitive(match); - if (case_sensitive != nullptr) { - route->matchers.path_matcher.case_sensitive = - google_protobuf_BoolValue_value(case_sensitive); +grpc_error_handle RoutePathMatchParse( + const envoy_config_route_v3_RouteMatch* match, XdsApi::Route* route, + bool* ignore_route) { + auto* case_sensitive_ptr = + envoy_config_route_v3_RouteMatch_case_sensitive(match); + bool case_sensitive = true; + if (case_sensitive_ptr != nullptr) { + case_sensitive = google_protobuf_BoolValue_value(case_sensitive_ptr); } + StringMatcher::Type type; + std::string match_string; if (envoy_config_route_v3_RouteMatch_has_prefix(match)) { absl::string_view prefix = UpbStringToAbsl(envoy_config_route_v3_RouteMatch_prefix(match)); @@ -1091,9 +1233,8 @@ grpc_error* RoutePathMatchParse(const envoy_config_route_v3_RouteMatch* match, return GRPC_ERROR_NONE; } } - route->matchers.path_matcher.type = - XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PREFIX; - route->matchers.path_matcher.string_matcher = std::string(prefix); + type = StringMatcher::Type::kPrefix; + match_string = std::string(prefix); } else if (envoy_config_route_v3_RouteMatch_has_path(match)) { absl::string_view path = UpbStringToAbsl(envoy_config_route_v3_RouteMatch_path(match)); @@ -1126,102 +1267,99 @@ grpc_error* RoutePathMatchParse(const envoy_config_route_v3_RouteMatch* match, *ignore_route = true; return GRPC_ERROR_NONE; } - route->matchers.path_matcher.type = - XdsApi::Route::Matchers::PathMatcher::PathMatcherType::PATH; - route->matchers.path_matcher.string_matcher = std::string(path); + type = StringMatcher::Type::kExact; + match_string = std::string(path); } else if (envoy_config_route_v3_RouteMatch_has_safe_regex(match)) { const envoy_type_matcher_v3_RegexMatcher* regex_matcher = envoy_config_route_v3_RouteMatch_safe_regex(match); GPR_ASSERT(regex_matcher != nullptr); - std::string matcher = UpbStringToStdString( + type = StringMatcher::Type::kSafeRegex; + match_string = UpbStringToStdString( envoy_type_matcher_v3_RegexMatcher_regex(regex_matcher)); - RE2::Options options; - options.set_case_sensitive(route->matchers.path_matcher.case_sensitive); - auto regex = absl::make_unique<RE2>(std::move(matcher), options); - if (!regex->ok()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Invalid regex string specified in path matcher."); - } - route->matchers.path_matcher.type = - XdsApi::Route::Matchers::PathMatcher::PathMatcherType::REGEX; - route->matchers.path_matcher.regex_matcher = std::move(regex); } else { return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Invalid route path specifier specified."); } + absl::StatusOr<StringMatcher> string_matcher = + StringMatcher::Create(type, match_string, case_sensitive); + if (!string_matcher.ok()) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("path matcher: ", string_matcher.status().message()) + .c_str()); + ; + } + route->matchers.path_matcher = std::move(string_matcher.value()); return GRPC_ERROR_NONE; } -grpc_error* RouteHeaderMatchersParse( +grpc_error_handle RouteHeaderMatchersParse( const envoy_config_route_v3_RouteMatch* match, XdsApi::Route* route) { size_t size; const envoy_config_route_v3_HeaderMatcher* const* headers = envoy_config_route_v3_RouteMatch_headers(match, &size); for (size_t i = 0; i < size; ++i) { const envoy_config_route_v3_HeaderMatcher* header = headers[i]; - XdsApi::Route::Matchers::HeaderMatcher header_matcher; - header_matcher.name = + const std::string name = UpbStringToStdString(envoy_config_route_v3_HeaderMatcher_name(header)); + HeaderMatcher::Type type; + std::string match_string; + int64_t range_start = 0; + int64_t range_end = 0; + bool present_match = false; if (envoy_config_route_v3_HeaderMatcher_has_exact_match(header)) { - header_matcher.type = - XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::EXACT; - header_matcher.string_matcher = UpbStringToStdString( + type = HeaderMatcher::Type::kExact; + match_string = UpbStringToStdString( envoy_config_route_v3_HeaderMatcher_exact_match(header)); } else if (envoy_config_route_v3_HeaderMatcher_has_safe_regex_match( header)) { const envoy_type_matcher_v3_RegexMatcher* regex_matcher = envoy_config_route_v3_HeaderMatcher_safe_regex_match(header); GPR_ASSERT(regex_matcher != nullptr); - const std::string matcher = UpbStringToStdString( + type = HeaderMatcher::Type::kSafeRegex; + match_string = UpbStringToStdString( envoy_type_matcher_v3_RegexMatcher_regex(regex_matcher)); - std::unique_ptr<RE2> regex = absl::make_unique<RE2>(matcher); - if (!regex->ok()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Invalid regex string specified in header matcher."); - } - header_matcher.type = - XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::REGEX; - header_matcher.regex_match = std::move(regex); } else if (envoy_config_route_v3_HeaderMatcher_has_range_match(header)) { - header_matcher.type = - XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::RANGE; + type = HeaderMatcher::Type::kRange; const envoy_type_v3_Int64Range* range_matcher = envoy_config_route_v3_HeaderMatcher_range_match(header); - header_matcher.range_start = - envoy_type_v3_Int64Range_start(range_matcher); - header_matcher.range_end = envoy_type_v3_Int64Range_end(range_matcher); - if (header_matcher.range_end < header_matcher.range_start) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Invalid range header matcher specifier specified: end " - "cannot be smaller than start."); - } + range_start = envoy_type_v3_Int64Range_start(range_matcher); + range_end = envoy_type_v3_Int64Range_end(range_matcher); } else if (envoy_config_route_v3_HeaderMatcher_has_present_match(header)) { - header_matcher.type = - XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PRESENT; - header_matcher.present_match = - envoy_config_route_v3_HeaderMatcher_present_match(header); + type = HeaderMatcher::Type::kPresent; + present_match = envoy_config_route_v3_HeaderMatcher_present_match(header); } else if (envoy_config_route_v3_HeaderMatcher_has_prefix_match(header)) { - header_matcher.type = - XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::PREFIX; - header_matcher.string_matcher = UpbStringToStdString( + type = HeaderMatcher::Type::kPrefix; + match_string = UpbStringToStdString( envoy_config_route_v3_HeaderMatcher_prefix_match(header)); } else if (envoy_config_route_v3_HeaderMatcher_has_suffix_match(header)) { - header_matcher.type = - XdsApi::Route::Matchers::HeaderMatcher::HeaderMatcherType::SUFFIX; - header_matcher.string_matcher = UpbStringToStdString( + type = HeaderMatcher::Type::kSuffix; + match_string = UpbStringToStdString( envoy_config_route_v3_HeaderMatcher_suffix_match(header)); + } else if (envoy_config_route_v3_HeaderMatcher_has_contains_match(header)) { + type = HeaderMatcher::Type::kContains; + match_string = UpbStringToStdString( + envoy_config_route_v3_HeaderMatcher_contains_match(header)); } else { return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "Invalid route header matcher specified."); } - header_matcher.invert_match = + bool invert_match = envoy_config_route_v3_HeaderMatcher_invert_match(header); - route->matchers.header_matchers.emplace_back(std::move(header_matcher)); + absl::StatusOr<HeaderMatcher> header_matcher = + HeaderMatcher::Create(name, type, match_string, range_start, range_end, + present_match, invert_match); + if (!header_matcher.ok()) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("header matcher: ", header_matcher.status().message()) + .c_str()); + } + route->matchers.header_matchers.emplace_back( + std::move(header_matcher.value())); } return GRPC_ERROR_NONE; } -grpc_error* RouteRuntimeFractionParse( +grpc_error_handle RouteRuntimeFractionParse( const envoy_config_route_v3_RouteMatch* match, XdsApi::Route* route) { const envoy_config_core_v3_RuntimeFractionalPercent* runtime_fraction = envoy_config_route_v3_RouteMatch_runtime_fraction(match); @@ -1254,8 +1392,98 @@ grpc_error* RouteRuntimeFractionParse( return GRPC_ERROR_NONE; } -grpc_error* RouteActionParse(const envoy_config_route_v3_Route* route_msg, - XdsApi::Route* route, bool* ignore_route) { +grpc_error_handle ExtractHttpFilterTypeName(const EncodingContext& context, + const google_protobuf_Any* any, + absl::string_view* filter_type) { + *filter_type = UpbStringToAbsl(google_protobuf_Any_type_url(any)); + if (*filter_type == "type.googleapis.com/udpa.type.v1.TypedStruct") { + upb_strview any_value = google_protobuf_Any_value(any); + const auto* typed_struct = udpa_type_v1_TypedStruct_parse( + any_value.data, any_value.size, context.arena); + if (typed_struct == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "could not parse TypedStruct from filter config"); + } + *filter_type = + UpbStringToAbsl(udpa_type_v1_TypedStruct_type_url(typed_struct)); + } + *filter_type = absl::StripPrefix(*filter_type, "type.googleapis.com/"); + return GRPC_ERROR_NONE; +} + +template <typename ParentType, typename EntryType> +grpc_error_handle ParseTypedPerFilterConfig( + const EncodingContext& context, const ParentType* parent, + const EntryType* (*entry_func)(const ParentType*, size_t*), + upb_strview (*key_func)(const EntryType*), + const google_protobuf_Any* (*value_func)(const EntryType*), + XdsApi::TypedPerFilterConfig* typed_per_filter_config) { + size_t filter_it = UPB_MAP_BEGIN; + while (true) { + const auto* filter_entry = entry_func(parent, &filter_it); + if (filter_entry == nullptr) break; + absl::string_view key = UpbStringToAbsl(key_func(filter_entry)); + if (key.empty()) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("empty filter name in map"); + } + const google_protobuf_Any* any = value_func(filter_entry); + GPR_ASSERT(any != nullptr); + absl::string_view filter_type = + UpbStringToAbsl(google_protobuf_Any_type_url(any)); + if (filter_type.empty()) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("no filter config specified for filter name ", key) + .c_str()); + } + bool is_optional = false; + if (filter_type == + "type.googleapis.com/envoy.config.route.v3.FilterConfig") { + upb_strview any_value = google_protobuf_Any_value(any); + const auto* filter_config = envoy_config_route_v3_FilterConfig_parse( + any_value.data, any_value.size, context.arena); + if (filter_config == nullptr) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("could not parse FilterConfig wrapper for ", key) + .c_str()); + } + is_optional = + envoy_config_route_v3_FilterConfig_is_optional(filter_config); + any = envoy_config_route_v3_FilterConfig_config(filter_config); + if (any == nullptr) { + if (is_optional) continue; + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("no filter config specified for filter name ", key) + .c_str()); + } + } + grpc_error_handle error = + ExtractHttpFilterTypeName(context, any, &filter_type); + if (error != GRPC_ERROR_NONE) return error; + const XdsHttpFilterImpl* filter_impl = + XdsHttpFilterRegistry::GetFilterForType(filter_type); + if (filter_impl == nullptr) { + if (is_optional) continue; + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("no filter registered for config type ", filter_type) + .c_str()); + } + absl::StatusOr<XdsHttpFilterImpl::FilterConfig> filter_config = + filter_impl->GenerateFilterConfigOverride( + google_protobuf_Any_value(any), context.arena); + if (!filter_config.ok()) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("filter config for type ", filter_type, + " failed to parse: ", filter_config.status().ToString()) + .c_str()); + } + (*typed_per_filter_config)[std::string(key)] = std::move(*filter_config); + } + return GRPC_ERROR_NONE; +} + +grpc_error_handle RouteActionParse(const EncodingContext& context, + const envoy_config_route_v3_Route* route_msg, + XdsApi::Route* route, bool* ignore_route) { if (!envoy_config_route_v3_Route_has_route(route_msg)) { return GRPC_ERROR_CREATE_FROM_STATIC_STRING( "No RouteAction found in route."); @@ -1307,6 +1535,17 @@ grpc_error* RouteActionParse(const envoy_config_route_v3_Route* route_msg, cluster.weight = google_protobuf_UInt32Value_value(weight); if (cluster.weight == 0) continue; sum_of_weights += cluster.weight; + if (context.use_v3) { + grpc_error_handle error = ParseTypedPerFilterConfig< + envoy_config_route_v3_WeightedCluster_ClusterWeight, + envoy_config_route_v3_WeightedCluster_ClusterWeight_TypedPerFilterConfigEntry>( + context, cluster_weight, + envoy_config_route_v3_WeightedCluster_ClusterWeight_typed_per_filter_config_next, + envoy_config_route_v3_WeightedCluster_ClusterWeight_TypedPerFilterConfigEntry_key, + envoy_config_route_v3_WeightedCluster_ClusterWeight_TypedPerFilterConfigEntry_value, + &cluster.typed_per_filter_config); + if (error != GRPC_ERROR_NONE) return error; + } route->weighted_clusters.emplace_back(std::move(cluster)); } if (total_weight != sum_of_weights) { @@ -1321,7 +1560,7 @@ grpc_error* RouteActionParse(const envoy_config_route_v3_Route* route_msg, // No cluster or weighted_clusters found in RouteAction, ignore this route. *ignore_route = true; } - if (XdsTimeoutEnabled() && !*ignore_route) { + if (!*ignore_route) { const envoy_config_route_v3_RouteAction_MaxStreamDuration* max_stream_duration = envoy_config_route_v3_RouteAction_max_stream_duration(route_action); @@ -1342,20 +1581,102 @@ grpc_error* RouteActionParse(const envoy_config_route_v3_Route* route_msg, } } } + // Get HashPolicy from RouteAction + if (XdsRingHashEnabled()) { + size_t size = 0; + const envoy_config_route_v3_RouteAction_HashPolicy* const* hash_policies = + envoy_config_route_v3_RouteAction_hash_policy(route_action, &size); + for (size_t i = 0; i < size; ++i) { + const envoy_config_route_v3_RouteAction_HashPolicy* hash_policy = + hash_policies[i]; + XdsApi::Route::HashPolicy policy; + policy.terminal = + envoy_config_route_v3_RouteAction_HashPolicy_terminal(hash_policy); + const envoy_config_route_v3_RouteAction_HashPolicy_Header* header; + const envoy_config_route_v3_RouteAction_HashPolicy_FilterState* + filter_state; + if ((header = envoy_config_route_v3_RouteAction_HashPolicy_header( + hash_policy)) != nullptr) { + policy.type = XdsApi::Route::HashPolicy::Type::HEADER; + policy.header_name = UpbStringToStdString( + envoy_config_route_v3_RouteAction_HashPolicy_Header_header_name( + header)); + const struct envoy_type_matcher_v3_RegexMatchAndSubstitute* + regex_rewrite = + envoy_config_route_v3_RouteAction_HashPolicy_Header_regex_rewrite( + header); + if (regex_rewrite == nullptr) { + gpr_log( + GPR_DEBUG, + "RouteAction HashPolicy contains policy specifier Header with " + "RegexMatchAndSubstitution but Regex is missing"); + continue; + } + const envoy_type_matcher_v3_RegexMatcher* regex_matcher = + envoy_type_matcher_v3_RegexMatchAndSubstitute_pattern( + regex_rewrite); + if (regex_matcher == nullptr) { + gpr_log( + GPR_DEBUG, + "RouteAction HashPolicy contains policy specifier Header with " + "RegexMatchAndSubstitution but RegexMatcher pattern is " + "missing"); + continue; + } + RE2::Options options; + policy.regex = absl::make_unique<RE2>( + UpbStringToStdString( + envoy_type_matcher_v3_RegexMatcher_regex(regex_matcher)), + options); + if (!policy.regex->ok()) { + gpr_log( + GPR_DEBUG, + "RouteAction HashPolicy contains policy specifier Header with " + "RegexMatchAndSubstitution but RegexMatcher pattern does not " + "compile"); + continue; + } + policy.regex_substitution = UpbStringToStdString( + envoy_type_matcher_v3_RegexMatchAndSubstitute_substitution( + regex_rewrite)); + } else if ((filter_state = + envoy_config_route_v3_RouteAction_HashPolicy_filter_state( + hash_policy)) != nullptr) { + std::string key = UpbStringToStdString( + envoy_config_route_v3_RouteAction_HashPolicy_FilterState_key( + filter_state)); + if (key == "io.grpc.channel_id") { + policy.type = XdsApi::Route::HashPolicy::Type::CHANNEL_ID; + } else { + gpr_log(GPR_DEBUG, + "RouteAction HashPolicy contains policy specifier " + "FilterState but " + "key is not io.grpc.channel_id."); + continue; + } + } else { + gpr_log( + GPR_DEBUG, + "RouteAction HashPolicy contains unsupported policy specifier."); + continue; + } + route->hash_policies.emplace_back(std::move(policy)); + } + } return GRPC_ERROR_NONE; } -grpc_error* RouteConfigParse( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, +grpc_error_handle RouteConfigParse( + const EncodingContext& context, const envoy_config_route_v3_RouteConfiguration* route_config, XdsApi::RdsUpdate* rds_update) { - MaybeLogRouteConfiguration(client, tracer, symtab, route_config); + MaybeLogRouteConfiguration(context, route_config); // Get the virtual hosts. - size_t size; + size_t num_virtual_hosts; const envoy_config_route_v3_VirtualHost* const* virtual_hosts = - envoy_config_route_v3_RouteConfiguration_virtual_hosts(route_config, - &size); - for (size_t i = 0; i < size; ++i) { + envoy_config_route_v3_RouteConfiguration_virtual_hosts( + route_config, &num_virtual_hosts); + for (size_t i = 0; i < num_virtual_hosts; ++i) { rds_update->virtual_hosts.emplace_back(); XdsApi::RdsUpdate::VirtualHost& vhost = rds_update->virtual_hosts.back(); // Parse domains. @@ -1375,6 +1696,18 @@ grpc_error* RouteConfigParse( if (vhost.domains.empty()) { return GRPC_ERROR_CREATE_FROM_STATIC_STRING("VirtualHost has no domains"); } + // Parse typed_per_filter_config. + if (context.use_v3) { + grpc_error_handle error = ParseTypedPerFilterConfig< + envoy_config_route_v3_VirtualHost, + envoy_config_route_v3_VirtualHost_TypedPerFilterConfigEntry>( + context, virtual_hosts[i], + envoy_config_route_v3_VirtualHost_typed_per_filter_config_next, + envoy_config_route_v3_VirtualHost_TypedPerFilterConfigEntry_key, + envoy_config_route_v3_VirtualHost_TypedPerFilterConfigEntry_value, + &vhost.typed_per_filter_config); + if (error != GRPC_ERROR_NONE) return error; + } // Parse routes. size_t num_routes; const envoy_config_route_v3_Route* const* routes = @@ -1387,6 +1720,9 @@ grpc_error* RouteConfigParse( for (size_t j = 0; j < num_routes; ++j) { const envoy_config_route_v3_RouteMatch* match = envoy_config_route_v3_Route_match(routes[j]); + if (match == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Match can't be null."); + } size_t query_parameters_size; static_cast<void>(envoy_config_route_v3_RouteMatch_query_parameters( match, &query_parameters_size)); @@ -1395,16 +1731,28 @@ grpc_error* RouteConfigParse( } XdsApi::Route route; bool ignore_route = false; - grpc_error* error = RoutePathMatchParse(match, &route, &ignore_route); + grpc_error_handle error = + RoutePathMatchParse(match, &route, &ignore_route); if (error != GRPC_ERROR_NONE) return error; if (ignore_route) continue; error = RouteHeaderMatchersParse(match, &route); if (error != GRPC_ERROR_NONE) return error; error = RouteRuntimeFractionParse(match, &route); if (error != GRPC_ERROR_NONE) return error; - error = RouteActionParse(routes[j], &route, &ignore_route); + error = RouteActionParse(context, routes[j], &route, &ignore_route); if (error != GRPC_ERROR_NONE) return error; if (ignore_route) continue; + if (context.use_v3) { + grpc_error_handle error = ParseTypedPerFilterConfig< + envoy_config_route_v3_Route, + envoy_config_route_v3_Route_TypedPerFilterConfigEntry>( + context, routes[j], + envoy_config_route_v3_Route_typed_per_filter_config_next, + envoy_config_route_v3_Route_TypedPerFilterConfigEntry_key, + envoy_config_route_v3_Route_TypedPerFilterConfigEntry_value, + &route.typed_per_filter_config); + if (error != GRPC_ERROR_NONE) return error; + } vhost.routes.emplace_back(std::move(route)); } if (vhost.routes.empty()) { @@ -1414,170 +1762,6 @@ grpc_error* RouteConfigParse( return GRPC_ERROR_NONE; } -grpc_error* LdsResponseParse( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, - const envoy_service_discovery_v3_DiscoveryResponse* response, - const std::set<absl::string_view>& expected_listener_names, - XdsApi::LdsUpdateMap* lds_update_map, upb_arena* arena) { - // Get the resources from the response. - size_t size; - const google_protobuf_Any* const* resources = - envoy_service_discovery_v3_DiscoveryResponse_resources(response, &size); - for (size_t i = 0; i < size; ++i) { - // Check the type_url of the resource. - absl::string_view type_url = - UpbStringToAbsl(google_protobuf_Any_type_url(resources[i])); - if (!IsLds(type_url)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Resource is not LDS."); - } - // Decode the listener. - const upb_strview encoded_listener = - google_protobuf_Any_value(resources[i]); - const envoy_config_listener_v3_Listener* listener = - envoy_config_listener_v3_Listener_parse(encoded_listener.data, - encoded_listener.size, arena); - if (listener == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Can't decode listener."); - } - // Check listener name. Ignore unexpected listeners. - std::string listener_name = - UpbStringToStdString(envoy_config_listener_v3_Listener_name(listener)); - if (expected_listener_names.find(listener_name) == - expected_listener_names.end()) { - continue; - } - // Fail if listener name is duplicated. - if (lds_update_map->find(listener_name) != lds_update_map->end()) { - return GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrCat("duplicate listener name \"", listener_name, "\"") - .c_str()); - } - XdsApi::LdsUpdate& lds_update = (*lds_update_map)[listener_name]; - // Get api_listener and decode it to http_connection_manager. - const envoy_config_listener_v3_ApiListener* api_listener = - envoy_config_listener_v3_Listener_api_listener(listener); - if (api_listener == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Listener has no ApiListener."); - } - const upb_strview encoded_api_listener = google_protobuf_Any_value( - envoy_config_listener_v3_ApiListener_api_listener(api_listener)); - const envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager* - http_connection_manager = - envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_parse( - encoded_api_listener.data, encoded_api_listener.size, arena); - if (http_connection_manager == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Could not parse HttpConnectionManager config from ApiListener"); - } - if (XdsTimeoutEnabled()) { - // Obtain max_stream_duration from Http Protocol Options. - const envoy_config_core_v3_HttpProtocolOptions* options = - envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_common_http_protocol_options( - http_connection_manager); - if (options != nullptr) { - const google_protobuf_Duration* duration = - envoy_config_core_v3_HttpProtocolOptions_max_stream_duration( - options); - if (duration != nullptr) { - lds_update.http_max_stream_duration.seconds = - google_protobuf_Duration_seconds(duration); - lds_update.http_max_stream_duration.nanos = - google_protobuf_Duration_nanos(duration); - } - } - } - // Found inlined route_config. Parse it to find the cluster_name. - if (envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_has_route_config( - http_connection_manager)) { - const envoy_config_route_v3_RouteConfiguration* route_config = - envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_route_config( - http_connection_manager); - XdsApi::RdsUpdate rds_update; - grpc_error* error = - RouteConfigParse(client, tracer, symtab, route_config, &rds_update); - if (error != GRPC_ERROR_NONE) return error; - lds_update.rds_update = std::move(rds_update); - continue; - } - // Validate that RDS must be used to get the route_config dynamically. - if (!envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_has_rds( - http_connection_manager)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "HttpConnectionManager neither has inlined route_config nor RDS."); - } - const envoy_extensions_filters_network_http_connection_manager_v3_Rds* rds = - envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_rds( - http_connection_manager); - // Check that the ConfigSource specifies ADS. - const envoy_config_core_v3_ConfigSource* config_source = - envoy_extensions_filters_network_http_connection_manager_v3_Rds_config_source( - rds); - if (config_source == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "HttpConnectionManager missing config_source for RDS."); - } - if (!envoy_config_core_v3_ConfigSource_has_ads(config_source)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "HttpConnectionManager ConfigSource for RDS does not specify ADS."); - } - // Get the route_config_name. - lds_update.route_config_name = UpbStringToStdString( - envoy_extensions_filters_network_http_connection_manager_v3_Rds_route_config_name( - rds)); - } - return GRPC_ERROR_NONE; -} - -grpc_error* RdsResponseParse( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, - const envoy_service_discovery_v3_DiscoveryResponse* response, - const std::set<absl::string_view>& expected_route_configuration_names, - XdsApi::RdsUpdateMap* rds_update_map, upb_arena* arena) { - // Get the resources from the response. - size_t size; - const google_protobuf_Any* const* resources = - envoy_service_discovery_v3_DiscoveryResponse_resources(response, &size); - for (size_t i = 0; i < size; ++i) { - // Check the type_url of the resource. - absl::string_view type_url = - UpbStringToAbsl(google_protobuf_Any_type_url(resources[i])); - if (!IsRds(type_url)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Resource is not RDS."); - } - // Decode the route_config. - const upb_strview encoded_route_config = - google_protobuf_Any_value(resources[i]); - const envoy_config_route_v3_RouteConfiguration* route_config = - envoy_config_route_v3_RouteConfiguration_parse( - encoded_route_config.data, encoded_route_config.size, arena); - if (route_config == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Can't decode route_config."); - } - // Check route_config_name. Ignore unexpected route_config. - std::string route_config_name = UpbStringToStdString( - envoy_config_route_v3_RouteConfiguration_name(route_config)); - if (expected_route_configuration_names.find(route_config_name) == - expected_route_configuration_names.end()) { - continue; - } - // Fail if route config name is duplicated. - if (rds_update_map->find(route_config_name) != rds_update_map->end()) { - return GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrCat("duplicate route config name \"", route_config_name, - "\"") - .c_str()); - } - // Parse the route_config. - XdsApi::RdsUpdate& rds_update = - (*rds_update_map)[std::move(route_config_name)]; - grpc_error* error = - RouteConfigParse(client, tracer, symtab, route_config, &rds_update); - if (error != GRPC_ERROR_NONE) return error; - } - return GRPC_ERROR_NONE; -} - XdsApi::CommonTlsContext::CertificateProviderInstance CertificateProviderInstanceParse( const envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_CertificateProviderInstance* @@ -1591,11 +1775,11 @@ CertificateProviderInstanceParse( certificate_provider_instance_proto))}; } -grpc_error* CommonTlsContextParse( +grpc_error_handle CommonTlsContextParse( const envoy_extensions_transport_sockets_tls_v3_CommonTlsContext* common_tls_context_proto, XdsApi::CommonTlsContext* common_tls_context) GRPC_MUST_USE_RESULT; -grpc_error* CommonTlsContextParse( +grpc_error_handle CommonTlsContextParse( const envoy_extensions_transport_sockets_tls_v3_CommonTlsContext* common_tls_context_proto, XdsApi::CommonTlsContext* common_tls_context) { @@ -1612,35 +1796,35 @@ grpc_error* CommonTlsContextParse( envoy_extensions_transport_sockets_tls_v3_CertificateValidationContext_match_subject_alt_names( default_validation_context, &len); for (size_t i = 0; i < len; ++i) { - XdsApi::StringMatcher::StringMatcherType type; + StringMatcher::Type type; std::string matcher; if (envoy_type_matcher_v3_StringMatcher_has_exact( subject_alt_names_matchers[i])) { - type = XdsApi::StringMatcher::StringMatcherType::EXACT; + type = StringMatcher::Type::kExact; matcher = UpbStringToStdString(envoy_type_matcher_v3_StringMatcher_exact( subject_alt_names_matchers[i])); } else if (envoy_type_matcher_v3_StringMatcher_has_prefix( subject_alt_names_matchers[i])) { - type = XdsApi::StringMatcher::StringMatcherType::PREFIX; + type = StringMatcher::Type::kPrefix; matcher = UpbStringToStdString(envoy_type_matcher_v3_StringMatcher_prefix( subject_alt_names_matchers[i])); } else if (envoy_type_matcher_v3_StringMatcher_has_suffix( subject_alt_names_matchers[i])) { - type = XdsApi::StringMatcher::StringMatcherType::SUFFIX; + type = StringMatcher::Type::kSuffix; matcher = UpbStringToStdString(envoy_type_matcher_v3_StringMatcher_suffix( subject_alt_names_matchers[i])); } else if (envoy_type_matcher_v3_StringMatcher_has_contains( subject_alt_names_matchers[i])) { - type = XdsApi::StringMatcher::StringMatcherType::CONTAINS; + type = StringMatcher::Type::kContains; matcher = UpbStringToStdString(envoy_type_matcher_v3_StringMatcher_contains( subject_alt_names_matchers[i])); } else if (envoy_type_matcher_v3_StringMatcher_has_safe_regex( subject_alt_names_matchers[i])) { - type = XdsApi::StringMatcher::StringMatcherType::SAFE_REGEX; + type = StringMatcher::Type::kSafeRegex; auto* regex_matcher = envoy_type_matcher_v3_StringMatcher_safe_regex( subject_alt_names_matchers[i]); matcher = UpbStringToStdString( @@ -1651,20 +1835,22 @@ grpc_error* CommonTlsContextParse( } bool ignore_case = envoy_type_matcher_v3_StringMatcher_ignore_case( subject_alt_names_matchers[i]); - XdsApi::StringMatcher string_matcher(type, matcher, ignore_case); - if (type == XdsApi::StringMatcher::StringMatcherType::SAFE_REGEX) { - if (!string_matcher.regex_matcher()->ok()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Invalid regex string specified in string matcher."); - } - if (ignore_case) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "StringMatcher: ignore_case has no effect for SAFE_REGEX."); - } + absl::StatusOr<StringMatcher> string_matcher = + StringMatcher::Create(type, matcher, + /*case_sensitive=*/!ignore_case); + if (!string_matcher.ok()) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("string matcher: ", + string_matcher.status().message()) + .c_str()); + } + if (type == StringMatcher::Type::kSafeRegex && ignore_case) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "StringMatcher: ignore_case has no effect for SAFE_REGEX."); } common_tls_context->combined_validation_context .default_validation_context.match_subject_alt_names.push_back( - std::move(string_matcher)); + std::move(string_matcher.value())); } } auto* validation_context_certificate_provider_instance = @@ -1688,11 +1874,797 @@ grpc_error* CommonTlsContextParse( return GRPC_ERROR_NONE; } -grpc_error* CdsResponseParse( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, +grpc_error_handle HttpConnectionManagerParse( + bool is_client, const EncodingContext& context, + const envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager* + http_connection_manager_proto, + bool is_v2, + XdsApi::LdsUpdate::HttpConnectionManager* http_connection_manager) { + MaybeLogHttpConnectionManager(context, http_connection_manager_proto); + // Obtain max_stream_duration from Http Protocol Options. + const envoy_config_core_v3_HttpProtocolOptions* options = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_common_http_protocol_options( + http_connection_manager_proto); + if (options != nullptr) { + const google_protobuf_Duration* duration = + envoy_config_core_v3_HttpProtocolOptions_max_stream_duration(options); + if (duration != nullptr) { + http_connection_manager->http_max_stream_duration.seconds = + google_protobuf_Duration_seconds(duration); + http_connection_manager->http_max_stream_duration.nanos = + google_protobuf_Duration_nanos(duration); + } + } + // Parse filters. + if (!is_v2) { + size_t num_filters = 0; + const auto* http_filters = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_http_filters( + http_connection_manager_proto, &num_filters); + std::set<absl::string_view> names_seen; + for (size_t i = 0; i < num_filters; ++i) { + const auto* http_filter = http_filters[i]; + absl::string_view name = UpbStringToAbsl( + envoy_extensions_filters_network_http_connection_manager_v3_HttpFilter_name( + http_filter)); + if (name.empty()) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("empty filter name at index ", i).c_str()); + } + if (names_seen.find(name) != names_seen.end()) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("duplicate HTTP filter name: ", name).c_str()); + } + names_seen.insert(name); + const bool is_optional = + envoy_extensions_filters_network_http_connection_manager_v3_HttpFilter_is_optional( + http_filter); + const google_protobuf_Any* any = + envoy_extensions_filters_network_http_connection_manager_v3_HttpFilter_typed_config( + http_filter); + if (any == nullptr) { + if (is_optional) continue; + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("no filter config specified for filter name ", name) + .c_str()); + } + absl::string_view filter_type; + grpc_error_handle error = + ExtractHttpFilterTypeName(context, any, &filter_type); + if (error != GRPC_ERROR_NONE) return error; + const XdsHttpFilterImpl* filter_impl = + XdsHttpFilterRegistry::GetFilterForType(filter_type); + if (filter_impl == nullptr) { + if (is_optional) continue; + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("no filter registered for config type ", filter_type) + .c_str()); + } + if ((is_client && !filter_impl->IsSupportedOnClients()) || + (!is_client && !filter_impl->IsSupportedOnServers())) { + if (is_optional) continue; + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrFormat("Filter %s is not supported on %s", filter_type, + is_client ? "clients" : "servers") + .c_str()); + } + absl::StatusOr<XdsHttpFilterImpl::FilterConfig> filter_config = + filter_impl->GenerateFilterConfig(google_protobuf_Any_value(any), + context.arena); + if (!filter_config.ok()) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat( + "filter config for type ", filter_type, + " failed to parse: ", filter_config.status().ToString()) + .c_str()); + } + http_connection_manager->http_filters.emplace_back( + XdsApi::LdsUpdate::HttpConnectionManager::HttpFilter{ + std::string(name), std::move(*filter_config)}); + } + } else { + // If using a v2 config, we just hard-code a list containing only the + // router filter without actually looking at the config. This ensures + // that the right thing happens in the xds resolver without having + // to expose whether the resource we received was v2 or v3. + http_connection_manager->http_filters.emplace_back( + XdsApi::LdsUpdate::HttpConnectionManager::HttpFilter{ + "router", {kXdsHttpRouterFilterConfigName, Json()}}); + } + if (is_client) { + // Found inlined route_config. Parse it to find the cluster_name. + if (envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_has_route_config( + http_connection_manager_proto)) { + const envoy_config_route_v3_RouteConfiguration* route_config = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_route_config( + http_connection_manager_proto); + XdsApi::RdsUpdate rds_update; + grpc_error_handle error = + RouteConfigParse(context, route_config, &rds_update); + if (error != GRPC_ERROR_NONE) return error; + http_connection_manager->rds_update = std::move(rds_update); + return GRPC_ERROR_NONE; + } + // Validate that RDS must be used to get the route_config dynamically. + const envoy_extensions_filters_network_http_connection_manager_v3_Rds* rds = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_rds( + http_connection_manager_proto); + if (rds == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "HttpConnectionManager neither has inlined route_config nor RDS."); + } + // Check that the ConfigSource specifies ADS. + const envoy_config_core_v3_ConfigSource* config_source = + envoy_extensions_filters_network_http_connection_manager_v3_Rds_config_source( + rds); + if (config_source == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "HttpConnectionManager missing config_source for RDS."); + } + if (!envoy_config_core_v3_ConfigSource_has_ads(config_source)) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "HttpConnectionManager ConfigSource for RDS does not specify ADS."); + } + // Get the route_config_name. + http_connection_manager->route_config_name = UpbStringToStdString( + envoy_extensions_filters_network_http_connection_manager_v3_Rds_route_config_name( + rds)); + } + return GRPC_ERROR_NONE; +} + +grpc_error_handle LdsResponseParseClient( + const EncodingContext& context, + const envoy_config_listener_v3_ApiListener* api_listener, bool is_v2, + XdsApi::LdsUpdate* lds_update) { + lds_update->type = XdsApi::LdsUpdate::ListenerType::kHttpApiListener; + const upb_strview encoded_api_listener = google_protobuf_Any_value( + envoy_config_listener_v3_ApiListener_api_listener(api_listener)); + const auto* http_connection_manager = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_parse( + encoded_api_listener.data, encoded_api_listener.size, context.arena); + if (http_connection_manager == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Could not parse HttpConnectionManager config from ApiListener"); + } + return HttpConnectionManagerParse(true /* is_client */, context, + http_connection_manager, is_v2, + &lds_update->http_connection_manager); +} + +grpc_error_handle DownstreamTlsContextParse( + const EncodingContext& context, + const envoy_config_core_v3_TransportSocket* transport_socket, + XdsApi::DownstreamTlsContext* downstream_tls_context) { + absl::string_view name = UpbStringToAbsl( + envoy_config_core_v3_TransportSocket_name(transport_socket)); + if (name == "envoy.transport_sockets.tls") { + auto* typed_config = + envoy_config_core_v3_TransportSocket_typed_config(transport_socket); + if (typed_config != nullptr) { + const upb_strview encoded_downstream_tls_context = + google_protobuf_Any_value(typed_config); + auto* downstream_tls_context_proto = + envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_parse( + encoded_downstream_tls_context.data, + encoded_downstream_tls_context.size, context.arena); + if (downstream_tls_context_proto == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Can't decode downstream tls context."); + } + auto* common_tls_context = + envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_common_tls_context( + downstream_tls_context_proto); + if (common_tls_context != nullptr) { + grpc_error_handle error = CommonTlsContextParse( + common_tls_context, &downstream_tls_context->common_tls_context); + if (error != GRPC_ERROR_NONE) return error; + } + auto* require_client_certificate = + envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_require_client_certificate( + downstream_tls_context_proto); + if (require_client_certificate != nullptr) { + downstream_tls_context->require_client_certificate = + google_protobuf_BoolValue_value(require_client_certificate); + } + } + if (downstream_tls_context->common_tls_context + .tls_certificate_certificate_provider_instance.instance_name + .empty()) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "TLS configuration provided but no " + "tls_certificate_certificate_provider_instance found."); + } + } + return GRPC_ERROR_NONE; +} + +grpc_error_handle CidrRangeParse( + const envoy_config_core_v3_CidrRange* cidr_range_proto, + XdsApi::LdsUpdate::FilterChainMap::CidrRange* cidr_range) { + std::string address_prefix = UpbStringToStdString( + envoy_config_core_v3_CidrRange_address_prefix(cidr_range_proto)); + grpc_error_handle error = + grpc_string_to_sockaddr(&cidr_range->address, address_prefix.c_str(), 0); + if (error != GRPC_ERROR_NONE) return error; + cidr_range->prefix_len = 0; + auto* prefix_len_proto = + envoy_config_core_v3_CidrRange_prefix_len(cidr_range_proto); + if (prefix_len_proto != nullptr) { + cidr_range->prefix_len = std::min( + google_protobuf_UInt32Value_value(prefix_len_proto), + (reinterpret_cast<const grpc_sockaddr*>(cidr_range->address.addr)) + ->sa_family == GRPC_AF_INET + ? uint32_t(32) + : uint32_t(128)); + } + // Normalize the network address by masking it with prefix_len + grpc_sockaddr_mask_bits(&cidr_range->address, cidr_range->prefix_len); + return GRPC_ERROR_NONE; +} + +grpc_error_handle FilterChainMatchParse( + const envoy_config_listener_v3_FilterChainMatch* filter_chain_match_proto, + FilterChain::FilterChainMatch* filter_chain_match) { + auto* destination_port = + envoy_config_listener_v3_FilterChainMatch_destination_port( + filter_chain_match_proto); + if (destination_port != nullptr) { + filter_chain_match->destination_port = + google_protobuf_UInt32Value_value(destination_port); + } + size_t size = 0; + auto* prefix_ranges = envoy_config_listener_v3_FilterChainMatch_prefix_ranges( + filter_chain_match_proto, &size); + filter_chain_match->prefix_ranges.reserve(size); + for (size_t i = 0; i < size; i++) { + XdsApi::LdsUpdate::FilterChainMap::CidrRange cidr_range; + grpc_error_handle error = CidrRangeParse(prefix_ranges[i], &cidr_range); + if (error != GRPC_ERROR_NONE) return error; + filter_chain_match->prefix_ranges.push_back(cidr_range); + } + filter_chain_match->source_type = + static_cast<XdsApi::LdsUpdate::FilterChainMap::ConnectionSourceType>( + envoy_config_listener_v3_FilterChainMatch_source_type( + filter_chain_match_proto)); + auto* source_prefix_ranges = + envoy_config_listener_v3_FilterChainMatch_source_prefix_ranges( + filter_chain_match_proto, &size); + filter_chain_match->source_prefix_ranges.reserve(size); + for (size_t i = 0; i < size; i++) { + XdsApi::LdsUpdate::FilterChainMap::CidrRange cidr_range; + grpc_error_handle error = + CidrRangeParse(source_prefix_ranges[i], &cidr_range); + if (error != GRPC_ERROR_NONE) return error; + filter_chain_match->source_prefix_ranges.push_back(cidr_range); + } + auto* source_ports = envoy_config_listener_v3_FilterChainMatch_source_ports( + filter_chain_match_proto, &size); + filter_chain_match->source_ports.reserve(size); + for (size_t i = 0; i < size; i++) { + filter_chain_match->source_ports.push_back(source_ports[i]); + } + auto* server_names = envoy_config_listener_v3_FilterChainMatch_server_names( + filter_chain_match_proto, &size); + for (size_t i = 0; i < size; i++) { + filter_chain_match->server_names.push_back( + UpbStringToStdString(server_names[i])); + } + filter_chain_match->transport_protocol = UpbStringToStdString( + envoy_config_listener_v3_FilterChainMatch_transport_protocol( + filter_chain_match_proto)); + auto* application_protocols = + envoy_config_listener_v3_FilterChainMatch_application_protocols( + filter_chain_match_proto, &size); + for (size_t i = 0; i < size; i++) { + filter_chain_match->application_protocols.push_back( + UpbStringToStdString(application_protocols[i])); + } + return GRPC_ERROR_NONE; +} + +grpc_error_handle FilterChainParse( + const EncodingContext& context, + const envoy_config_listener_v3_FilterChain* filter_chain_proto, bool is_v2, + FilterChain* filter_chain) { + grpc_error_handle error = GRPC_ERROR_NONE; + auto* filter_chain_match = + envoy_config_listener_v3_FilterChain_filter_chain_match( + filter_chain_proto); + if (filter_chain_match != nullptr) { + error = FilterChainMatchParse(filter_chain_match, + &filter_chain->filter_chain_match); + if (error != GRPC_ERROR_NONE) return error; + } + // Parse the filters list. Currently we only support HttpConnectionManager. + size_t size = 0; + auto* filters = + envoy_config_listener_v3_FilterChain_filters(filter_chain_proto, &size); + if (size != 1) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "FilterChain should have exactly one filter: HttpConnectionManager; no " + "other filter is supported at the moment"); + } + auto* typed_config = envoy_config_listener_v3_Filter_typed_config(filters[0]); + if (typed_config == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "No typed_config found in filter."); + } + absl::string_view type_url = + UpbStringToAbsl(google_protobuf_Any_type_url(typed_config)); + if (type_url != + "type.googleapis.com/" + "envoy.extensions.filters.network.http_connection_manager.v3." + "HttpConnectionManager") { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("Unsupported filter type ", type_url).c_str()); + } + const upb_strview encoded_http_connection_manager = + google_protobuf_Any_value(typed_config); + const auto* http_connection_manager = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_parse( + encoded_http_connection_manager.data, + encoded_http_connection_manager.size, context.arena); + if (http_connection_manager == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Could not parse HttpConnectionManager config from filter " + "typed_config"); + } + filter_chain->filter_chain_data = + std::make_shared<XdsApi::LdsUpdate::FilterChainData>(); + error = HttpConnectionManagerParse( + false /* is_client */, context, http_connection_manager, is_v2, + &filter_chain->filter_chain_data->http_connection_manager); + if (error != GRPC_ERROR_NONE) return error; + // Get the DownstreamTlsContext for the filter chain + if (XdsSecurityEnabled()) { + auto* transport_socket = + envoy_config_listener_v3_FilterChain_transport_socket( + filter_chain_proto); + if (transport_socket != nullptr) { + error = DownstreamTlsContextParse( + context, transport_socket, + &filter_chain->filter_chain_data->downstream_tls_context); + } + } + return error; +} + +grpc_error_handle AddressParse( + const envoy_config_core_v3_Address* address_proto, std::string* address) { + const auto* socket_address = + envoy_config_core_v3_Address_socket_address(address_proto); + if (socket_address == nullptr) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + "Address does not have socket_address"); + } + if (envoy_config_core_v3_SocketAddress_protocol(socket_address) != + envoy_config_core_v3_SocketAddress_TCP) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "SocketAddress protocol is not TCP"); + } + uint32_t port = envoy_config_core_v3_SocketAddress_port_value(socket_address); + if (port > 65535) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Invalid port"); + } + *address = JoinHostPort( + UpbStringToAbsl( + envoy_config_core_v3_SocketAddress_address(socket_address)), + port); + return GRPC_ERROR_NONE; +} + +// An intermediate map for filter chains that we create to validate the list of +// filter chains received from the control plane and to finally create +// XdsApi::LdsUpdate::FilterChainMap +struct InternalFilterChainMap { + using SourceIpMap = + std::map<std::string, XdsApi::LdsUpdate::FilterChainMap::SourceIp>; + using ConnectionSourceTypesArray = std::array<SourceIpMap, 3>; + struct DestinationIp { + absl::optional<XdsApi::LdsUpdate::FilterChainMap::CidrRange> prefix_range; + bool transport_protocol_raw_buffer_provided = false; + ConnectionSourceTypesArray source_types_array; + }; + using DestinationIpMap = std::map<std::string, DestinationIp>; + DestinationIpMap destination_ip_map; +}; + +grpc_error_handle AddFilterChainDataForSourcePort( + const FilterChain& filter_chain, + XdsApi::LdsUpdate::FilterChainMap::SourcePortsMap* ports_map, + uint32_t port) { + auto insert_result = ports_map->emplace( + port, XdsApi::LdsUpdate::FilterChainMap::FilterChainDataSharedPtr{ + filter_chain.filter_chain_data}); + if (!insert_result.second) { + return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat( + "Duplicate matching rules detected when adding filter chain: ", + filter_chain.filter_chain_match.ToString()) + .c_str()); + } + return GRPC_ERROR_NONE; +} + +grpc_error_handle AddFilterChainDataForSourcePorts( + const FilterChain& filter_chain, + XdsApi::LdsUpdate::FilterChainMap::SourcePortsMap* ports_map) { + if (filter_chain.filter_chain_match.source_ports.empty()) { + return AddFilterChainDataForSourcePort(filter_chain, ports_map, 0); + } else { + for (uint32_t port : filter_chain.filter_chain_match.source_ports) { + grpc_error_handle error = + AddFilterChainDataForSourcePort(filter_chain, ports_map, port); + if (error != GRPC_ERROR_NONE) return error; + } + } + return GRPC_ERROR_NONE; +} + +grpc_error_handle AddFilterChainDataForSourceIpRange( + const FilterChain& filter_chain, + InternalFilterChainMap::SourceIpMap* source_ip_map) { + if (filter_chain.filter_chain_match.source_prefix_ranges.empty()) { + auto insert_result = source_ip_map->emplace( + "", XdsApi::LdsUpdate::FilterChainMap::SourceIp()); + return AddFilterChainDataForSourcePorts( + filter_chain, &insert_result.first->second.ports_map); + } else { + for (const auto& prefix_range : + filter_chain.filter_chain_match.source_prefix_ranges) { + auto insert_result = source_ip_map->emplace( + absl::StrCat(grpc_sockaddr_to_string(&prefix_range.address, false), + "/", prefix_range.prefix_len), + XdsApi::LdsUpdate::FilterChainMap::SourceIp()); + if (insert_result.second) { + insert_result.first->second.prefix_range.emplace(prefix_range); + } + grpc_error_handle error = AddFilterChainDataForSourcePorts( + filter_chain, &insert_result.first->second.ports_map); + if (error != GRPC_ERROR_NONE) return error; + } + } + return GRPC_ERROR_NONE; +} + +grpc_error_handle AddFilterChainDataForSourceType( + const FilterChain& filter_chain, + InternalFilterChainMap::DestinationIp* destination_ip) { + GPR_ASSERT(static_cast<unsigned int>( + filter_chain.filter_chain_match.source_type) < 3); + return AddFilterChainDataForSourceIpRange( + filter_chain, &destination_ip->source_types_array[static_cast<int>( + filter_chain.filter_chain_match.source_type)]); +} + +grpc_error_handle AddFilterChainDataForApplicationProtocols( + const FilterChain& filter_chain, + InternalFilterChainMap::DestinationIp* destination_ip) { + // Only allow filter chains that do not mention application protocols + if (!filter_chain.filter_chain_match.application_protocols.empty()) { + return GRPC_ERROR_NONE; + } + return AddFilterChainDataForSourceType(filter_chain, destination_ip); +} + +grpc_error_handle AddFilterChainDataForTransportProtocol( + const FilterChain& filter_chain, + InternalFilterChainMap::DestinationIp* destination_ip) { + const std::string& transport_protocol = + filter_chain.filter_chain_match.transport_protocol; + // Only allow filter chains with no transport protocol or "raw_buffer" + if (!transport_protocol.empty() && transport_protocol != "raw_buffer") { + return GRPC_ERROR_NONE; + } + // If for this configuration, we've already seen filter chains that mention + // the transport protocol as "raw_buffer", we will never match filter chains + // that do not mention it. + if (destination_ip->transport_protocol_raw_buffer_provided && + transport_protocol.empty()) { + return GRPC_ERROR_NONE; + } + if (!transport_protocol.empty() && + !destination_ip->transport_protocol_raw_buffer_provided) { + destination_ip->transport_protocol_raw_buffer_provided = true; + // Clear out the previous entries if any since those entries did not mention + // "raw_buffer" + destination_ip->source_types_array = + InternalFilterChainMap::ConnectionSourceTypesArray(); + } + return AddFilterChainDataForApplicationProtocols(filter_chain, + destination_ip); +} + +grpc_error_handle AddFilterChainDataForServerNames( + const FilterChain& filter_chain, + InternalFilterChainMap::DestinationIp* destination_ip) { + // Don't continue adding filter chains with server names mentioned + if (!filter_chain.filter_chain_match.server_names.empty()) { + return GRPC_ERROR_NONE; + } + return AddFilterChainDataForTransportProtocol(filter_chain, destination_ip); +} + +grpc_error_handle AddFilterChainDataForDestinationIpRange( + const FilterChain& filter_chain, + InternalFilterChainMap::DestinationIpMap* destination_ip_map) { + if (filter_chain.filter_chain_match.prefix_ranges.empty()) { + auto insert_result = destination_ip_map->emplace( + "", InternalFilterChainMap::DestinationIp()); + return AddFilterChainDataForServerNames(filter_chain, + &insert_result.first->second); + } else { + for (const auto& prefix_range : + filter_chain.filter_chain_match.prefix_ranges) { + auto insert_result = destination_ip_map->emplace( + absl::StrCat(grpc_sockaddr_to_string(&prefix_range.address, false), + "/", prefix_range.prefix_len), + InternalFilterChainMap::DestinationIp()); + if (insert_result.second) { + insert_result.first->second.prefix_range.emplace(prefix_range); + } + grpc_error_handle error = AddFilterChainDataForServerNames( + filter_chain, &insert_result.first->second); + if (error != GRPC_ERROR_NONE) return error; + } + } + return GRPC_ERROR_NONE; +} + +XdsApi::LdsUpdate::FilterChainMap BuildFromInternalFilterChainMap( + InternalFilterChainMap* internal_filter_chain_map) { + XdsApi::LdsUpdate::FilterChainMap filter_chain_map; + for (auto& destination_ip_pair : + internal_filter_chain_map->destination_ip_map) { + XdsApi::LdsUpdate::FilterChainMap::DestinationIp destination_ip; + destination_ip.prefix_range = destination_ip_pair.second.prefix_range; + for (int i = 0; i < 3; i++) { + auto& source_ip_map = destination_ip_pair.second.source_types_array[i]; + for (auto& source_ip_pair : source_ip_map) { + destination_ip.source_types_array[i].push_back( + std::move(source_ip_pair.second)); + } + } + filter_chain_map.destination_ip_vector.push_back(std::move(destination_ip)); + } + return filter_chain_map; +} + +grpc_error_handle BuildFilterChainMap( + const std::vector<FilterChain>& filter_chains, + XdsApi::LdsUpdate::FilterChainMap* filter_chain_map) { + InternalFilterChainMap internal_filter_chain_map; + for (const auto& filter_chain : filter_chains) { + // Discard filter chain entries that specify destination port + if (filter_chain.filter_chain_match.destination_port != 0) continue; + grpc_error_handle error = AddFilterChainDataForDestinationIpRange( + filter_chain, &internal_filter_chain_map.destination_ip_map); + if (error != GRPC_ERROR_NONE) return error; + } + *filter_chain_map = + BuildFromInternalFilterChainMap(&internal_filter_chain_map); + return GRPC_ERROR_NONE; +} + +grpc_error_handle LdsResponseParseServer( + const EncodingContext& context, + const envoy_config_listener_v3_Listener* listener, bool is_v2, + XdsApi::LdsUpdate* lds_update) { + lds_update->type = XdsApi::LdsUpdate::ListenerType::kTcpListener; + grpc_error_handle error = + AddressParse(envoy_config_listener_v3_Listener_address(listener), + &lds_update->address); + if (error != GRPC_ERROR_NONE) return error; + const auto* use_original_dst = + envoy_config_listener_v3_Listener_use_original_dst(listener); + if (use_original_dst != nullptr) { + if (google_protobuf_BoolValue_value(use_original_dst)) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Field \'use_original_dst\' is not supported."); + } + } + size_t size = 0; + auto* filter_chains = + envoy_config_listener_v3_Listener_filter_chains(listener, &size); + std::vector<FilterChain> parsed_filter_chains; + parsed_filter_chains.reserve(size); + for (size_t i = 0; i < size; i++) { + FilterChain filter_chain; + error = FilterChainParse(context, filter_chains[i], is_v2, &filter_chain); + if (error != GRPC_ERROR_NONE) return error; + parsed_filter_chains.push_back(std::move(filter_chain)); + } + error = + BuildFilterChainMap(parsed_filter_chains, &lds_update->filter_chain_map); + if (error != GRPC_ERROR_NONE) return error; + auto* default_filter_chain = + envoy_config_listener_v3_Listener_default_filter_chain(listener); + if (default_filter_chain != nullptr) { + FilterChain filter_chain; + error = + FilterChainParse(context, default_filter_chain, is_v2, &filter_chain); + if (error != GRPC_ERROR_NONE) return error; + if (filter_chain.filter_chain_data != nullptr) { + lds_update->default_filter_chain = + std::move(*filter_chain.filter_chain_data); + } + } + if (size == 0 && default_filter_chain == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("No filter chain provided."); + } + return GRPC_ERROR_NONE; +} + +grpc_error_handle LdsResponseParse( + const EncodingContext& context, + const envoy_service_discovery_v3_DiscoveryResponse* response, + const std::set<absl::string_view>& expected_listener_names, + XdsApi::LdsUpdateMap* lds_update_map, + std::set<std::string>* resource_names_failed) { + std::vector<grpc_error_handle> errors; + // Get the resources from the response. + size_t size; + const google_protobuf_Any* const* resources = + envoy_service_discovery_v3_DiscoveryResponse_resources(response, &size); + for (size_t i = 0; i < size; ++i) { + // Check the type_url of the resource. + absl::string_view type_url = + UpbStringToAbsl(google_protobuf_Any_type_url(resources[i])); + bool is_v2 = false; + if (!IsLds(type_url, &is_v2)) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("resource index ", i, ": Resource is not LDS.") + .c_str())); + continue; + } + // Decode the listener. + const upb_strview encoded_listener = + google_protobuf_Any_value(resources[i]); + const envoy_config_listener_v3_Listener* listener = + envoy_config_listener_v3_Listener_parse( + encoded_listener.data, encoded_listener.size, context.arena); + if (listener == nullptr) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("resource index ", i, ": Can't decode listener.") + .c_str())); + continue; + } + // Check listener name. Ignore unexpected listeners. + std::string listener_name = + UpbStringToStdString(envoy_config_listener_v3_Listener_name(listener)); + if (expected_listener_names.find(listener_name) == + expected_listener_names.end()) { + continue; + } + // Fail if listener name is duplicated. + if (lds_update_map->find(listener_name) != lds_update_map->end()) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("duplicate listener name \"", listener_name, "\"") + .c_str())); + resource_names_failed->insert(listener_name); + continue; + } + // Serialize into JSON and store it in the LdsUpdateMap + XdsApi::LdsResourceData& lds_resource_data = + (*lds_update_map)[listener_name]; + XdsApi::LdsUpdate& lds_update = lds_resource_data.resource; + lds_resource_data.serialized_proto = UpbStringToStdString(encoded_listener); + // Check whether it's a client or server listener. + const envoy_config_listener_v3_ApiListener* api_listener = + envoy_config_listener_v3_Listener_api_listener(listener); + const envoy_config_core_v3_Address* address = + envoy_config_listener_v3_Listener_address(listener); + if (api_listener != nullptr && address != nullptr) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(listener_name, + ": Listener has both address and ApiListener") + .c_str())); + resource_names_failed->insert(listener_name); + continue; + } + if (api_listener == nullptr && address == nullptr) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(listener_name, + ": Listener has neither address nor ApiListener") + .c_str())); + resource_names_failed->insert(listener_name); + continue; + } + grpc_error_handle error = GRPC_ERROR_NONE; + if (api_listener != nullptr) { + error = LdsResponseParseClient(context, api_listener, is_v2, &lds_update); + } else { + error = LdsResponseParseServer(context, listener, is_v2, &lds_update); + } + if (error != GRPC_ERROR_NONE) { + errors.push_back(grpc_error_add_child( + GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(listener_name, ": validation error").c_str()), + error)); + resource_names_failed->insert(listener_name); + } + } + return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing LDS response", &errors); +} + +grpc_error_handle RdsResponseParse( + const EncodingContext& context, + const envoy_service_discovery_v3_DiscoveryResponse* response, + const std::set<absl::string_view>& expected_route_configuration_names, + XdsApi::RdsUpdateMap* rds_update_map, + std::set<std::string>* resource_names_failed) { + std::vector<grpc_error_handle> errors; + // Get the resources from the response. + size_t size; + const google_protobuf_Any* const* resources = + envoy_service_discovery_v3_DiscoveryResponse_resources(response, &size); + for (size_t i = 0; i < size; ++i) { + // Check the type_url of the resource. + absl::string_view type_url = + UpbStringToAbsl(google_protobuf_Any_type_url(resources[i])); + if (!IsRds(type_url)) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("resource index ", i, ": Resource is not RDS.") + .c_str())); + continue; + } + // Decode the route_config. + const upb_strview encoded_route_config = + google_protobuf_Any_value(resources[i]); + const envoy_config_route_v3_RouteConfiguration* route_config = + envoy_config_route_v3_RouteConfiguration_parse( + encoded_route_config.data, encoded_route_config.size, + context.arena); + if (route_config == nullptr) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("resource index ", i, ": Can't decode route_config.") + .c_str())); + continue; + } + // Check route_config_name. Ignore unexpected route_config. + std::string route_config_name = UpbStringToStdString( + envoy_config_route_v3_RouteConfiguration_name(route_config)); + if (expected_route_configuration_names.find(route_config_name) == + expected_route_configuration_names.end()) { + continue; + } + // Fail if route config name is duplicated. + if (rds_update_map->find(route_config_name) != rds_update_map->end()) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("duplicate route config name \"", route_config_name, + "\"") + .c_str())); + resource_names_failed->insert(route_config_name); + continue; + } + // Serialize into JSON and store it in the RdsUpdateMap + XdsApi::RdsResourceData& rds_resource_data = + (*rds_update_map)[route_config_name]; + XdsApi::RdsUpdate& rds_update = rds_resource_data.resource; + rds_resource_data.serialized_proto = + UpbStringToStdString(encoded_route_config); + // Parse the route_config. + grpc_error_handle error = + RouteConfigParse(context, route_config, &rds_update); + if (error != GRPC_ERROR_NONE) { + errors.push_back(grpc_error_add_child( + GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(route_config_name, ": validation error").c_str()), + error)); + resource_names_failed->insert(route_config_name); + } + } + return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing RDS response", &errors); +} + +grpc_error_handle CdsResponseParse( + const EncodingContext& context, const envoy_service_discovery_v3_DiscoveryResponse* response, const std::set<absl::string_view>& expected_cluster_names, - XdsApi::CdsUpdateMap* cds_update_map, upb_arena* arena) { + XdsApi::CdsUpdateMap* cds_update_map, + std::set<std::string>* resource_names_failed) { + std::vector<grpc_error_handle> errors; // Get the resources from the response. size_t size; const google_protobuf_Any* const* resources = @@ -1703,17 +2675,23 @@ grpc_error* CdsResponseParse( absl::string_view type_url = UpbStringToAbsl(google_protobuf_Any_type_url(resources[i])); if (!IsCds(type_url)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Resource is not CDS."); + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("resource index ", i, ": Resource is not CDS.") + .c_str())); + continue; } // Decode the cluster. const upb_strview encoded_cluster = google_protobuf_Any_value(resources[i]); const envoy_config_cluster_v3_Cluster* cluster = - envoy_config_cluster_v3_Cluster_parse(encoded_cluster.data, - encoded_cluster.size, arena); + envoy_config_cluster_v3_Cluster_parse( + encoded_cluster.data, encoded_cluster.size, context.arena); if (cluster == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Can't decode cluster."); + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("resource index ", i, ": Can't decode cluster.") + .c_str())); + continue; } - MaybeLogCluster(client, tracer, symtab, cluster); + MaybeLogCluster(context, cluster); // Ignore unexpected cluster names. std::string cluster_name = UpbStringToStdString(envoy_config_cluster_v3_Cluster_name(cluster)); @@ -1723,41 +2701,194 @@ grpc_error* CdsResponseParse( } // Fail on duplicate resources. if (cds_update_map->find(cluster_name) != cds_update_map->end()) { - return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrCat("duplicate resource name \"", cluster_name, "\"") - .c_str()); + .c_str())); + resource_names_failed->insert(cluster_name); + continue; } - XdsApi::CdsUpdate& cds_update = (*cds_update_map)[std::move(cluster_name)]; + // Serialize into JSON and store it in the CdsUpdateMap + XdsApi::CdsResourceData& cds_resource_data = + (*cds_update_map)[cluster_name]; + XdsApi::CdsUpdate& cds_update = cds_resource_data.resource; + cds_resource_data.serialized_proto = UpbStringToStdString(encoded_cluster); // Check the cluster_discovery_type. - if (!envoy_config_cluster_v3_Cluster_has_type(cluster)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("DiscoveryType not found."); + if (!envoy_config_cluster_v3_Cluster_has_type(cluster) && + !envoy_config_cluster_v3_Cluster_has_cluster_type(cluster)) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, ": DiscoveryType not found.").c_str())); + resource_names_failed->insert(cluster_name); + continue; } - if (envoy_config_cluster_v3_Cluster_type(cluster) != + if (envoy_config_cluster_v3_Cluster_type(cluster) == envoy_config_cluster_v3_Cluster_EDS) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("DiscoveryType is not EDS."); - } - // Check the EDS config source. - const envoy_config_cluster_v3_Cluster_EdsClusterConfig* eds_cluster_config = - envoy_config_cluster_v3_Cluster_eds_cluster_config(cluster); - const envoy_config_core_v3_ConfigSource* eds_config = - envoy_config_cluster_v3_Cluster_EdsClusterConfig_eds_config( - eds_cluster_config); - if (!envoy_config_core_v3_ConfigSource_has_ads(eds_config)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "EDS ConfigSource is not ADS."); - } - // Record EDS service_name (if any). - upb_strview service_name = - envoy_config_cluster_v3_Cluster_EdsClusterConfig_service_name( - eds_cluster_config); - if (service_name.size != 0) { - cds_update.eds_service_name = UpbStringToStdString(service_name); + cds_update.cluster_type = XdsApi::CdsUpdate::ClusterType::EDS; + // Check the EDS config source. + const envoy_config_cluster_v3_Cluster_EdsClusterConfig* + eds_cluster_config = + envoy_config_cluster_v3_Cluster_eds_cluster_config(cluster); + const envoy_config_core_v3_ConfigSource* eds_config = + envoy_config_cluster_v3_Cluster_EdsClusterConfig_eds_config( + eds_cluster_config); + if (!envoy_config_core_v3_ConfigSource_has_ads(eds_config)) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, ": EDS ConfigSource is not ADS.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; + } + // Record EDS service_name (if any). + upb_strview service_name = + envoy_config_cluster_v3_Cluster_EdsClusterConfig_service_name( + eds_cluster_config); + if (service_name.size != 0) { + cds_update.eds_service_name = UpbStringToStdString(service_name); + } + } else if (!XdsAggregateAndLogicalDnsClusterEnabled()) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, ": DiscoveryType is not valid.").c_str())); + resource_names_failed->insert(cluster_name); + continue; + } else if (envoy_config_cluster_v3_Cluster_type(cluster) == + envoy_config_cluster_v3_Cluster_LOGICAL_DNS) { + cds_update.cluster_type = XdsApi::CdsUpdate::ClusterType::LOGICAL_DNS; + } else { + if (envoy_config_cluster_v3_Cluster_has_cluster_type(cluster)) { + const envoy_config_cluster_v3_Cluster_CustomClusterType* + custom_cluster_type = + envoy_config_cluster_v3_Cluster_cluster_type(cluster); + upb_strview type_name = + envoy_config_cluster_v3_Cluster_CustomClusterType_name( + custom_cluster_type); + if (UpbStringToAbsl(type_name) == "envoy.clusters.aggregate") { + cds_update.cluster_type = XdsApi::CdsUpdate::ClusterType::AGGREGATE; + // Retrieve aggregate clusters. + const google_protobuf_Any* typed_config = + envoy_config_cluster_v3_Cluster_CustomClusterType_typed_config( + custom_cluster_type); + const upb_strview aggregate_cluster_config_upb_strview = + google_protobuf_Any_value(typed_config); + const envoy_extensions_clusters_aggregate_v3_ClusterConfig* + aggregate_cluster_config = + envoy_extensions_clusters_aggregate_v3_ClusterConfig_parse( + aggregate_cluster_config_upb_strview.data, + aggregate_cluster_config_upb_strview.size, context.arena); + if (aggregate_cluster_config == nullptr) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, ": Can't parse aggregate cluster.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; + } + size_t size; + const upb_strview* clusters = + envoy_extensions_clusters_aggregate_v3_ClusterConfig_clusters( + aggregate_cluster_config, &size); + for (size_t i = 0; i < size; ++i) { + const upb_strview cluster = clusters[i]; + cds_update.prioritized_cluster_names.emplace_back( + UpbStringToStdString(cluster)); + } + } else { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, ": DiscoveryType is not valid.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; + } + } else { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, ": DiscoveryType is not valid.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; + } } // Check the LB policy. - if (envoy_config_cluster_v3_Cluster_lb_policy(cluster) != + if (envoy_config_cluster_v3_Cluster_lb_policy(cluster) == envoy_config_cluster_v3_Cluster_ROUND_ROBIN) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "LB policy is not ROUND_ROBIN."); + cds_update.lb_policy = "ROUND_ROBIN"; + } else if (XdsRingHashEnabled() && + envoy_config_cluster_v3_Cluster_lb_policy(cluster) == + envoy_config_cluster_v3_Cluster_RING_HASH) { + cds_update.lb_policy = "RING_HASH"; + // Record ring hash lb config + auto* ring_hash_config = + envoy_config_cluster_v3_Cluster_ring_hash_lb_config(cluster); + if (ring_hash_config == nullptr) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, + ": ring hash lb config required but not present.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; + } + const google_protobuf_UInt64Value* max_ring_size = + envoy_config_cluster_v3_Cluster_RingHashLbConfig_maximum_ring_size( + ring_hash_config); + if (max_ring_size != nullptr) { + cds_update.max_ring_size = + google_protobuf_UInt64Value_value(max_ring_size); + if (cds_update.max_ring_size > 8388608 || + cds_update.max_ring_size == 0) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat( + cluster_name, + ": max_ring_size is not in the range of 1 to 8388608.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; + } + } + const google_protobuf_UInt64Value* min_ring_size = + envoy_config_cluster_v3_Cluster_RingHashLbConfig_minimum_ring_size( + ring_hash_config); + if (min_ring_size != nullptr) { + cds_update.min_ring_size = + google_protobuf_UInt64Value_value(min_ring_size); + if (cds_update.min_ring_size > 8388608 || + cds_update.min_ring_size == 0) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat( + cluster_name, + ": min_ring_size is not in the range of 1 to 8388608.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; + } + if (cds_update.min_ring_size > cds_update.max_ring_size) { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat( + cluster_name, + ": min_ring_size cannot be greater than max_ring_size.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; + } + } + if (envoy_config_cluster_v3_Cluster_RingHashLbConfig_hash_function( + ring_hash_config) == + envoy_config_cluster_v3_Cluster_RingHashLbConfig_XX_HASH) { + cds_update.hash_function = XdsApi::CdsUpdate::HashFunction::XX_HASH; + } else if ( + envoy_config_cluster_v3_Cluster_RingHashLbConfig_hash_function( + ring_hash_config) == + envoy_config_cluster_v3_Cluster_RingHashLbConfig_MURMUR_HASH_2) { + cds_update.hash_function = + XdsApi::CdsUpdate::HashFunction::MURMUR_HASH_2; + } else { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, + ": ring hash lb config has invalid hash function.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; + } + } else { + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, ": LB policy is not supported.").c_str())); + resource_names_failed->insert(cluster_name); + continue; } if (XdsSecurityEnabled()) { // Record Upstream tls context @@ -1776,26 +2907,43 @@ grpc_error* CdsResponseParse( auto* upstream_tls_context = envoy_extensions_transport_sockets_tls_v3_UpstreamTlsContext_parse( encoded_upstream_tls_context.data, - encoded_upstream_tls_context.size, arena); + encoded_upstream_tls_context.size, context.arena); if (upstream_tls_context == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Can't decode upstream tls context."); + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, + ": Can't decode upstream tls context.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; } auto* common_tls_context = envoy_extensions_transport_sockets_tls_v3_UpstreamTlsContext_common_tls_context( upstream_tls_context); if (common_tls_context != nullptr) { - grpc_error* error = CommonTlsContextParse( + grpc_error_handle error = CommonTlsContextParse( common_tls_context, &cds_update.common_tls_context); - if (error != GRPC_ERROR_NONE) return error; + if (error != GRPC_ERROR_NONE) { + errors.push_back(grpc_error_add_child( + GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, ": error in TLS context") + .c_str()), + error)); + resource_names_failed->insert(cluster_name); + continue; + } } } if (cds_update.common_tls_context.combined_validation_context .validation_context_certificate_provider_instance .instance_name.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "TLS configuration provided but no " - "validation_context_certificate_provider_instance found."); + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, + "TLS configuration provided but no " + "validation_context_certificate_provider_instance " + "found.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; } } } @@ -1805,8 +2953,11 @@ grpc_error* CdsResponseParse( envoy_config_cluster_v3_Cluster_lrs_server(cluster); if (lrs_server != nullptr) { if (!envoy_config_core_v3_ConfigSource_has_self(lrs_server)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "LRS ConfigSource is not self."); + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(cluster_name, ": LRS ConfigSource is not self.") + .c_str())); + resource_names_failed->insert(cluster_name); + continue; } cds_update.lrs_load_reporting_server_name.emplace(""); } @@ -1837,10 +2988,10 @@ grpc_error* CdsResponseParse( } } } - return GRPC_ERROR_NONE; + return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing CDS response", &errors); } -grpc_error* ServerAddressParseAndAppend( +grpc_error_handle ServerAddressParseAndAppend( const envoy_config_endpoint_v3_LbEndpoint* lb_endpoint, ServerAddressList* list) { // If health_status is not HEALTHY or UNKNOWN, skip this endpoint. @@ -1865,13 +3016,15 @@ grpc_error* ServerAddressParseAndAppend( } // Populate grpc_resolved_address. grpc_resolved_address addr; - grpc_string_to_sockaddr(&addr, address_str.c_str(), port); + grpc_error_handle error = + grpc_string_to_sockaddr(&addr, address_str.c_str(), port); + if (error != GRPC_ERROR_NONE) return error; // Append the address to the list. list->emplace_back(addr, nullptr); return GRPC_ERROR_NONE; } -grpc_error* LocalityParse( +grpc_error_handle LocalityParse( const envoy_config_endpoint_v3_LocalityLbEndpoints* locality_lb_endpoints, XdsApi::EdsUpdate::Priority::Locality* output_locality, size_t* priority) { // Parse LB weight. @@ -1888,6 +3041,9 @@ grpc_error* LocalityParse( const envoy_config_core_v3_Locality* locality = envoy_config_endpoint_v3_LocalityLbEndpoints_locality( locality_lb_endpoints); + if (locality == nullptr) { + return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty locality."); + } std::string region = UpbStringToStdString(envoy_config_core_v3_Locality_region(locality)); std::string zone = @@ -1902,7 +3058,7 @@ grpc_error* LocalityParse( envoy_config_endpoint_v3_LocalityLbEndpoints_lb_endpoints( locality_lb_endpoints, &size); for (size_t i = 0; i < size; ++i) { - grpc_error* error = ServerAddressParseAndAppend( + grpc_error_handle error = ServerAddressParseAndAppend( lb_endpoints[i], &output_locality->endpoints); if (error != GRPC_ERROR_NONE) return error; } @@ -1912,7 +3068,7 @@ grpc_error* LocalityParse( return GRPC_ERROR_NONE; } -grpc_error* DropParseAndAppend( +grpc_error_handle DropParseAndAppend( const envoy_config_endpoint_v3_ClusterLoadAssignment_Policy_DropOverload* drop_overload, XdsApi::EdsUpdate::DropConfig* drop_config) { @@ -1951,11 +3107,13 @@ grpc_error* DropParseAndAppend( return GRPC_ERROR_NONE; } -grpc_error* EdsResponseParse( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, +grpc_error_handle EdsResponseParse( + const EncodingContext& context, const envoy_service_discovery_v3_DiscoveryResponse* response, const std::set<absl::string_view>& expected_eds_service_names, - XdsApi::EdsUpdateMap* eds_update_map, upb_arena* arena) { + XdsApi::EdsUpdateMap* eds_update_map, + std::set<std::string>* resource_names_failed) { + std::vector<grpc_error_handle> errors; // Get the resources from the response. size_t size; const google_protobuf_Any* const* resources = @@ -1965,7 +3123,10 @@ grpc_error* EdsResponseParse( absl::string_view type_url = UpbStringToAbsl(google_protobuf_Any_type_url(resources[i])); if (!IsEds(type_url)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Resource is not EDS."); + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("resource index ", i, ": Resource is not EDS.") + .c_str())); + continue; } // Get the cluster_load_assignment. upb_strview encoded_cluster_load_assignment = @@ -1973,13 +3134,15 @@ grpc_error* EdsResponseParse( envoy_config_endpoint_v3_ClusterLoadAssignment* cluster_load_assignment = envoy_config_endpoint_v3_ClusterLoadAssignment_parse( encoded_cluster_load_assignment.data, - encoded_cluster_load_assignment.size, arena); + encoded_cluster_load_assignment.size, context.arena); if (cluster_load_assignment == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Can't parse cluster_load_assignment."); + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat("resource index ", i, + ": Can't parse cluster_load_assignment.") + .c_str())); + continue; } - MaybeLogClusterLoadAssignment(client, tracer, symtab, - cluster_load_assignment); + MaybeLogClusterLoadAssignment(context, cluster_load_assignment); // Check the EDS service name. Ignore unexpected names. std::string eds_service_name = UpbStringToStdString( envoy_config_endpoint_v3_ClusterLoadAssignment_cluster_name( @@ -1990,22 +3153,29 @@ grpc_error* EdsResponseParse( } // Fail on duplicate resources. if (eds_update_map->find(eds_service_name) != eds_update_map->end()) { - return GRPC_ERROR_CREATE_FROM_COPIED_STRING( + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrCat("duplicate resource name \"", eds_service_name, "\"") - .c_str()); + .c_str())); + resource_names_failed->insert(eds_service_name); + continue; } - XdsApi::EdsUpdate& eds_update = - (*eds_update_map)[std::move(eds_service_name)]; + // Serialize into JSON and store it in the EdsUpdateMap + XdsApi::EdsResourceData& eds_resource_data = + (*eds_update_map)[eds_service_name]; + XdsApi::EdsUpdate& eds_update = eds_resource_data.resource; + eds_resource_data.serialized_proto = + UpbStringToStdString(encoded_cluster_load_assignment); // Get the endpoints. size_t locality_size; const envoy_config_endpoint_v3_LocalityLbEndpoints* const* endpoints = envoy_config_endpoint_v3_ClusterLoadAssignment_endpoints( cluster_load_assignment, &locality_size); + grpc_error_handle error = GRPC_ERROR_NONE; for (size_t j = 0; j < locality_size; ++j) { size_t priority; XdsApi::EdsUpdate::Priority::Locality locality; - grpc_error* error = LocalityParse(endpoints[j], &locality, &priority); - if (error != GRPC_ERROR_NONE) return error; + error = LocalityParse(endpoints[j], &locality, &priority); + if (error != GRPC_ERROR_NONE) break; // Filter out locality with weight 0. if (locality.lb_weight == 0) continue; // Make sure prorities is big enough. Note that they might not @@ -2016,10 +3186,21 @@ grpc_error* EdsResponseParse( eds_update.priorities[priority].localities.emplace(locality.name.get(), std::move(locality)); } + if (error != GRPC_ERROR_NONE) { + errors.push_back(grpc_error_add_child( + GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(eds_service_name, ": locality validation error") + .c_str()), + error)); + resource_names_failed->insert(eds_service_name); + continue; + } for (const auto& priority : eds_update.priorities) { if (priority.localities.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "EDS update includes sparse priority list"); + errors.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(eds_service_name, ": sparse priority list").c_str())); + resource_names_failed->insert(eds_service_name); + continue; } } // Get the drop config. @@ -2034,13 +3215,22 @@ grpc_error* EdsResponseParse( envoy_config_endpoint_v3_ClusterLoadAssignment_Policy_drop_overloads( policy, &drop_size); for (size_t j = 0; j < drop_size; ++j) { - grpc_error* error = + error = DropParseAndAppend(drop_overload[j], eds_update.drop_config.get()); - if (error != GRPC_ERROR_NONE) return error; + if (error != GRPC_ERROR_NONE) break; + } + if (error != GRPC_ERROR_NONE) { + errors.push_back(grpc_error_add_child( + GRPC_ERROR_CREATE_FROM_COPIED_STRING( + absl::StrCat(eds_service_name, ": drop config validation error") + .c_str()), + error)); + resource_names_failed->insert(eds_service_name); + continue; } } } - return GRPC_ERROR_NONE; + return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing EDS response", &errors); } std::string TypeUrlInternalToExternal(absl::string_view type_url) { @@ -2056,16 +3246,27 @@ std::string TypeUrlInternalToExternal(absl::string_view type_url) { return std::string(type_url); } +template <typename UpdateMap> +void MoveUpdatesToFailedSet(UpdateMap* update_map, + std::set<std::string>* resource_names_failed) { + for (const auto& p : *update_map) { + resource_names_failed->insert(p.first); + } + update_map->clear(); +} + } // namespace XdsApi::AdsParseResult XdsApi::ParseAdsResponse( - const grpc_slice& encoded_response, + const XdsBootstrap::XdsServer& server, const grpc_slice& encoded_response, const std::set<absl::string_view>& expected_listener_names, const std::set<absl::string_view>& expected_route_configuration_names, const std::set<absl::string_view>& expected_cluster_names, const std::set<absl::string_view>& expected_eds_service_names) { AdsParseResult result; upb::Arena arena; + const EncodingContext context = {client_, tracer_, symtab_.ptr(), arena.ptr(), + server.ShouldUseV3()}; // Decode the response. const envoy_service_discovery_v3_DiscoveryResponse* response = envoy_service_discovery_v3_DiscoveryResponse_parse( @@ -2077,7 +3278,7 @@ XdsApi::AdsParseResult XdsApi::ParseAdsResponse( GRPC_ERROR_CREATE_FROM_STATIC_STRING("Can't decode DiscoveryResponse."); return result; } - MaybeLogDiscoveryResponse(client_, tracer_, symtab_.ptr(), response); + MaybeLogDiscoveryResponse(context, response); // Record the type_url, the version_info, and the nonce of the response. result.type_url = TypeUrlInternalToExternal(UpbStringToAbsl( envoy_service_discovery_v3_DiscoveryResponse_type_url(response))); @@ -2087,22 +3288,37 @@ XdsApi::AdsParseResult XdsApi::ParseAdsResponse( envoy_service_discovery_v3_DiscoveryResponse_nonce(response)); // Parse the response according to the resource type. if (IsLds(result.type_url)) { - result.parse_error = LdsResponseParse(client_, tracer_, symtab_.ptr(), - response, expected_listener_names, - &result.lds_update_map, arena.ptr()); + result.parse_error = + LdsResponseParse(context, response, expected_listener_names, + &result.lds_update_map, &result.resource_names_failed); + if (result.parse_error != GRPC_ERROR_NONE) { + MoveUpdatesToFailedSet(&result.lds_update_map, + &result.resource_names_failed); + } } else if (IsRds(result.type_url)) { result.parse_error = - RdsResponseParse(client_, tracer_, symtab_.ptr(), response, - expected_route_configuration_names, - &result.rds_update_map, arena.ptr()); + RdsResponseParse(context, response, expected_route_configuration_names, + &result.rds_update_map, &result.resource_names_failed); + if (result.parse_error != GRPC_ERROR_NONE) { + MoveUpdatesToFailedSet(&result.rds_update_map, + &result.resource_names_failed); + } } else if (IsCds(result.type_url)) { - result.parse_error = CdsResponseParse(client_, tracer_, symtab_.ptr(), - response, expected_cluster_names, - &result.cds_update_map, arena.ptr()); + result.parse_error = + CdsResponseParse(context, response, expected_cluster_names, + &result.cds_update_map, &result.resource_names_failed); + if (result.parse_error != GRPC_ERROR_NONE) { + MoveUpdatesToFailedSet(&result.cds_update_map, + &result.resource_names_failed); + } } else if (IsEds(result.type_url)) { - result.parse_error = EdsResponseParse(client_, tracer_, symtab_.ptr(), - response, expected_eds_service_names, - &result.eds_update_map, arena.ptr()); + result.parse_error = + EdsResponseParse(context, response, expected_eds_service_names, + &result.eds_update_map, &result.resource_names_failed); + if (result.parse_error != GRPC_ERROR_NONE) { + MoveUpdatesToFailedSet(&result.eds_update_map, + &result.resource_names_failed); + } } return result; } @@ -2110,25 +3326,25 @@ XdsApi::AdsParseResult XdsApi::ParseAdsResponse( namespace { void MaybeLogLrsRequest( - XdsClient* client, TraceFlag* tracer, upb_symtab* symtab, + const EncodingContext& context, const envoy_service_load_stats_v3_LoadStatsRequest* request) { - if (GRPC_TRACE_FLAG_ENABLED(*tracer) && + if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { const upb_msgdef* msg_type = - envoy_service_load_stats_v3_LoadStatsRequest_getmsgdef(symtab); + envoy_service_load_stats_v3_LoadStatsRequest_getmsgdef(context.symtab); char buf[10240]; upb_text_encode(request, msg_type, nullptr, 0, buf, sizeof(buf)); - gpr_log(GPR_DEBUG, "[xds_client %p] constructed LRS request: %s", client, - buf); + gpr_log(GPR_DEBUG, "[xds_client %p] constructed LRS request: %s", + context.client, buf); } } grpc_slice SerializeLrsRequest( - const envoy_service_load_stats_v3_LoadStatsRequest* request, - upb_arena* arena) { + const EncodingContext& context, + const envoy_service_load_stats_v3_LoadStatsRequest* request) { size_t output_length; char* output = envoy_service_load_stats_v3_LoadStatsRequest_serialize( - request, arena, &output_length); + request, context.arena, &output_length); return grpc_slice_from_copied_buffer(output, output_length); } @@ -2137,6 +3353,8 @@ grpc_slice SerializeLrsRequest( grpc_slice XdsApi::CreateLrsInitialRequest( const XdsBootstrap::XdsServer& server) { upb::Arena arena; + const EncodingContext context = {client_, tracer_, symtab_.ptr(), arena.ptr(), + server.ShouldUseV3()}; // Create a request. envoy_service_load_stats_v3_LoadStatsRequest* request = envoy_service_load_stats_v3_LoadStatsRequest_new(arena.ptr()); @@ -2144,25 +3362,25 @@ grpc_slice XdsApi::CreateLrsInitialRequest( envoy_config_core_v3_Node* node_msg = envoy_service_load_stats_v3_LoadStatsRequest_mutable_node(request, arena.ptr()); - PopulateNode(arena.ptr(), node_, server.ShouldUseV3(), build_version_, - user_agent_name_, node_msg); + PopulateNode(context, node_, build_version_, user_agent_name_, node_msg); envoy_config_core_v3_Node_add_client_features( node_msg, upb_strview_makez("envoy.lrs.supports_send_all_clusters"), arena.ptr()); - MaybeLogLrsRequest(client_, tracer_, symtab_.ptr(), request); - return SerializeLrsRequest(request, arena.ptr()); + MaybeLogLrsRequest(context, request); + return SerializeLrsRequest(context, request); } namespace { void LocalityStatsPopulate( + const EncodingContext& context, envoy_config_endpoint_v3_UpstreamLocalityStats* output, const XdsLocalityName& locality_name, - const XdsClusterLocalityStats::Snapshot& snapshot, upb_arena* arena) { + const XdsClusterLocalityStats::Snapshot& snapshot) { // Set locality. envoy_config_core_v3_Locality* locality = - envoy_config_endpoint_v3_UpstreamLocalityStats_mutable_locality(output, - arena); + envoy_config_endpoint_v3_UpstreamLocalityStats_mutable_locality( + output, context.arena); if (!locality_name.region().empty()) { envoy_config_core_v3_Locality_set_region( locality, StdStringToUpbString(locality_name.region())); @@ -2190,7 +3408,7 @@ void LocalityStatsPopulate( const XdsClusterLocalityStats::BackendMetric& metric_value = p.second; envoy_config_endpoint_v3_EndpointLoadMetricStats* load_metric = envoy_config_endpoint_v3_UpstreamLocalityStats_add_load_metric_stats( - output, arena); + output, context.arena); envoy_config_endpoint_v3_EndpointLoadMetricStats_set_metric_name( load_metric, StdStringToUpbString(metric_name)); envoy_config_endpoint_v3_EndpointLoadMetricStats_set_num_requests_finished_with_metric( @@ -2205,6 +3423,8 @@ void LocalityStatsPopulate( grpc_slice XdsApi::CreateLrsRequest( ClusterLoadReportMap cluster_load_report_map) { upb::Arena arena; + const EncodingContext context = {client_, tracer_, symtab_.ptr(), arena.ptr(), + false}; // Create a request. envoy_service_load_stats_v3_LoadStatsRequest* request = envoy_service_load_stats_v3_LoadStatsRequest_new(arena.ptr()); @@ -2231,8 +3451,7 @@ grpc_slice XdsApi::CreateLrsRequest( envoy_config_endpoint_v3_UpstreamLocalityStats* locality_stats = envoy_config_endpoint_v3_ClusterStats_add_upstream_locality_stats( cluster_stats, arena.ptr()); - LocalityStatsPopulate(locality_stats, locality_name, snapshot, - arena.ptr()); + LocalityStatsPopulate(context, locality_stats, locality_name, snapshot); } // Add dropped requests. uint64_t total_dropped_requests = 0; @@ -2261,14 +3480,14 @@ grpc_slice XdsApi::CreateLrsRequest( google_protobuf_Duration_set_seconds(load_report_interval, timespec.tv_sec); google_protobuf_Duration_set_nanos(load_report_interval, timespec.tv_nsec); } - MaybeLogLrsRequest(client_, tracer_, symtab_.ptr(), request); - return SerializeLrsRequest(request, arena.ptr()); + MaybeLogLrsRequest(context, request); + return SerializeLrsRequest(context, request); } -grpc_error* XdsApi::ParseLrsResponse(const grpc_slice& encoded_response, - bool* send_all_clusters, - std::set<std::string>* cluster_names, - grpc_millis* load_reporting_interval) { +grpc_error_handle XdsApi::ParseLrsResponse( + const grpc_slice& encoded_response, bool* send_all_clusters, + std::set<std::string>* cluster_names, + grpc_millis* load_reporting_interval) { upb::Arena arena; // Decode the response. const envoy_service_load_stats_v3_LoadStatsResponse* decoded_response = @@ -2305,4 +3524,276 @@ grpc_error* XdsApi::ParseLrsResponse(const grpc_slice& encoded_response, return GRPC_ERROR_NONE; } +namespace { +google_protobuf_Timestamp* GrpcMillisToTimestamp(const EncodingContext& context, + grpc_millis value) { + google_protobuf_Timestamp* timestamp = + google_protobuf_Timestamp_new(context.arena); + gpr_timespec timespec = grpc_millis_to_timespec(value, GPR_CLOCK_REALTIME); + google_protobuf_Timestamp_set_seconds(timestamp, timespec.tv_sec); + google_protobuf_Timestamp_set_nanos(timestamp, timespec.tv_nsec); + return timestamp; +} + +envoy_admin_v3_UpdateFailureState* CreateUpdateFailureStateUpb( + const EncodingContext& context, + const XdsApi::ResourceMetadata* resource_metadata) { + auto* update_failure_state = + envoy_admin_v3_UpdateFailureState_new(context.arena); + envoy_admin_v3_UpdateFailureState_set_details( + update_failure_state, + StdStringToUpbString(resource_metadata->failed_details)); + envoy_admin_v3_UpdateFailureState_set_version_info( + update_failure_state, + StdStringToUpbString(resource_metadata->failed_version)); + envoy_admin_v3_UpdateFailureState_set_last_update_attempt( + update_failure_state, + GrpcMillisToTimestamp(context, resource_metadata->failed_update_time)); + return update_failure_state; +} + +void DumpLdsConfig(const EncodingContext& context, + const XdsApi::ResourceTypeMetadata& resource_type_metadata, + envoy_service_status_v3_PerXdsConfig* per_xds_config) { + upb_strview kLdsTypeUrlUpb = upb_strview_makez(XdsApi::kLdsTypeUrl); + auto* listener_config_dump = + envoy_service_status_v3_PerXdsConfig_mutable_listener_config( + per_xds_config, context.arena); + envoy_admin_v3_ListenersConfigDump_set_version_info( + listener_config_dump, + StdStringToUpbString(resource_type_metadata.version)); + for (auto& p : resource_type_metadata.resource_metadata_map) { + absl::string_view name = p.first; + const XdsApi::ResourceMetadata* meta = p.second; + const upb_strview name_upb = StdStringToUpbString(name); + auto* dynamic_listener = + envoy_admin_v3_ListenersConfigDump_add_dynamic_listeners( + listener_config_dump, context.arena); + envoy_admin_v3_ListenersConfigDump_DynamicListener_set_name( + dynamic_listener, name_upb); + envoy_admin_v3_ListenersConfigDump_DynamicListener_set_client_status( + dynamic_listener, meta->client_status); + if (!meta->serialized_proto.empty()) { + // Set in-effective listeners + auto* dynamic_listener_state = + envoy_admin_v3_ListenersConfigDump_DynamicListener_mutable_active_state( + dynamic_listener, context.arena); + envoy_admin_v3_ListenersConfigDump_DynamicListenerState_set_version_info( + dynamic_listener_state, StdStringToUpbString(meta->version)); + envoy_admin_v3_ListenersConfigDump_DynamicListenerState_set_last_updated( + dynamic_listener_state, + GrpcMillisToTimestamp(context, meta->update_time)); + auto* listener_any = + envoy_admin_v3_ListenersConfigDump_DynamicListenerState_mutable_listener( + dynamic_listener_state, context.arena); + google_protobuf_Any_set_type_url(listener_any, kLdsTypeUrlUpb); + google_protobuf_Any_set_value( + listener_any, StdStringToUpbString(meta->serialized_proto)); + } + if (meta->client_status == XdsApi::ResourceMetadata::NACKED) { + // Set error_state if NACKED + envoy_admin_v3_ListenersConfigDump_DynamicListener_set_error_state( + dynamic_listener, CreateUpdateFailureStateUpb(context, meta)); + } + } +} + +void DumpRdsConfig(const EncodingContext& context, + const XdsApi::ResourceTypeMetadata& resource_type_metadata, + envoy_service_status_v3_PerXdsConfig* per_xds_config) { + upb_strview kRdsTypeUrlUpb = upb_strview_makez(XdsApi::kRdsTypeUrl); + auto* route_config_dump = + envoy_service_status_v3_PerXdsConfig_mutable_route_config(per_xds_config, + context.arena); + for (auto& p : resource_type_metadata.resource_metadata_map) { + absl::string_view name = p.first; + const XdsApi::ResourceMetadata* meta = p.second; + const upb_strview name_upb = StdStringToUpbString(name); + auto* dynamic_route_config = + envoy_admin_v3_RoutesConfigDump_add_dynamic_route_configs( + route_config_dump, context.arena); + envoy_admin_v3_RoutesConfigDump_DynamicRouteConfig_set_client_status( + dynamic_route_config, meta->client_status); + auto* route_config_any = + envoy_admin_v3_RoutesConfigDump_DynamicRouteConfig_mutable_route_config( + dynamic_route_config, context.arena); + if (!meta->serialized_proto.empty()) { + // Set in-effective route configs + envoy_admin_v3_RoutesConfigDump_DynamicRouteConfig_set_version_info( + dynamic_route_config, StdStringToUpbString(meta->version)); + envoy_admin_v3_RoutesConfigDump_DynamicRouteConfig_set_last_updated( + dynamic_route_config, + GrpcMillisToTimestamp(context, meta->update_time)); + google_protobuf_Any_set_type_url(route_config_any, kRdsTypeUrlUpb); + google_protobuf_Any_set_value( + route_config_any, StdStringToUpbString(meta->serialized_proto)); + } else { + // If there isn't a working route config, we still need to print the + // name. + auto* route_config = + envoy_config_route_v3_RouteConfiguration_new(context.arena); + envoy_config_route_v3_RouteConfiguration_set_name(route_config, name_upb); + size_t length; + char* bytes = envoy_config_route_v3_RouteConfiguration_serialize( + route_config, context.arena, &length); + google_protobuf_Any_set_type_url(route_config_any, kRdsTypeUrlUpb); + google_protobuf_Any_set_value(route_config_any, + upb_strview_make(bytes, length)); + } + if (meta->client_status == XdsApi::ResourceMetadata::NACKED) { + // Set error_state if NACKED + envoy_admin_v3_RoutesConfigDump_DynamicRouteConfig_set_error_state( + dynamic_route_config, CreateUpdateFailureStateUpb(context, meta)); + } + } +} + +void DumpCdsConfig(const EncodingContext& context, + const XdsApi::ResourceTypeMetadata& resource_type_metadata, + envoy_service_status_v3_PerXdsConfig* per_xds_config) { + upb_strview kCdsTypeUrlUpb = upb_strview_makez(XdsApi::kCdsTypeUrl); + auto* cluster_config_dump = + envoy_service_status_v3_PerXdsConfig_mutable_cluster_config( + per_xds_config, context.arena); + envoy_admin_v3_ClustersConfigDump_set_version_info( + cluster_config_dump, + StdStringToUpbString(resource_type_metadata.version)); + for (auto& p : resource_type_metadata.resource_metadata_map) { + absl::string_view name = p.first; + const XdsApi::ResourceMetadata* meta = p.second; + const upb_strview name_upb = StdStringToUpbString(name); + auto* dynamic_cluster = + envoy_admin_v3_ClustersConfigDump_add_dynamic_active_clusters( + cluster_config_dump, context.arena); + envoy_admin_v3_ClustersConfigDump_DynamicCluster_set_client_status( + dynamic_cluster, meta->client_status); + auto* cluster_any = + envoy_admin_v3_ClustersConfigDump_DynamicCluster_mutable_cluster( + dynamic_cluster, context.arena); + if (!meta->serialized_proto.empty()) { + // Set in-effective clusters + envoy_admin_v3_ClustersConfigDump_DynamicCluster_set_version_info( + dynamic_cluster, StdStringToUpbString(meta->version)); + envoy_admin_v3_ClustersConfigDump_DynamicCluster_set_last_updated( + dynamic_cluster, GrpcMillisToTimestamp(context, meta->update_time)); + google_protobuf_Any_set_type_url(cluster_any, kCdsTypeUrlUpb); + google_protobuf_Any_set_value( + cluster_any, StdStringToUpbString(meta->serialized_proto)); + } else { + // If there isn't a working cluster, we still need to print the name. + auto* cluster = envoy_config_cluster_v3_Cluster_new(context.arena); + envoy_config_cluster_v3_Cluster_set_name(cluster, name_upb); + size_t length; + char* bytes = envoy_config_cluster_v3_Cluster_serialize( + cluster, context.arena, &length); + google_protobuf_Any_set_type_url(cluster_any, kCdsTypeUrlUpb); + google_protobuf_Any_set_value(cluster_any, + upb_strview_make(bytes, length)); + } + if (meta->client_status == XdsApi::ResourceMetadata::NACKED) { + // Set error_state if NACKED + envoy_admin_v3_ClustersConfigDump_DynamicCluster_set_error_state( + dynamic_cluster, CreateUpdateFailureStateUpb(context, meta)); + } + } +} + +void DumpEdsConfig(const EncodingContext& context, + const XdsApi::ResourceTypeMetadata& resource_type_metadata, + envoy_service_status_v3_PerXdsConfig* per_xds_config) { + upb_strview kEdsTypeUrlUpb = upb_strview_makez(XdsApi::kEdsTypeUrl); + auto* endpoint_config_dump = + envoy_service_status_v3_PerXdsConfig_mutable_endpoint_config( + per_xds_config, context.arena); + for (auto& p : resource_type_metadata.resource_metadata_map) { + absl::string_view name = p.first; + const XdsApi::ResourceMetadata* meta = p.second; + const upb_strview name_upb = StdStringToUpbString(name); + auto* dynamic_endpoint = + envoy_admin_v3_EndpointsConfigDump_add_dynamic_endpoint_configs( + endpoint_config_dump, context.arena); + envoy_admin_v3_EndpointsConfigDump_DynamicEndpointConfig_set_client_status( + dynamic_endpoint, meta->client_status); + auto* endpoint_any = + envoy_admin_v3_EndpointsConfigDump_DynamicEndpointConfig_mutable_endpoint_config( + dynamic_endpoint, context.arena); + if (!meta->serialized_proto.empty()) { + // Set in-effective endpoints + envoy_admin_v3_EndpointsConfigDump_DynamicEndpointConfig_set_version_info( + dynamic_endpoint, StdStringToUpbString(meta->version)); + envoy_admin_v3_EndpointsConfigDump_DynamicEndpointConfig_set_last_updated( + dynamic_endpoint, GrpcMillisToTimestamp(context, meta->update_time)); + google_protobuf_Any_set_type_url(endpoint_any, kEdsTypeUrlUpb); + google_protobuf_Any_set_value( + endpoint_any, StdStringToUpbString(meta->serialized_proto)); + } else { + // If there isn't a working endpoint, we still need to print the name. + auto* cluster_load_assignment = + envoy_config_endpoint_v3_ClusterLoadAssignment_new(context.arena); + envoy_config_endpoint_v3_ClusterLoadAssignment_set_cluster_name( + cluster_load_assignment, name_upb); + size_t length; + char* bytes = envoy_config_endpoint_v3_ClusterLoadAssignment_serialize( + cluster_load_assignment, context.arena, &length); + google_protobuf_Any_set_type_url(endpoint_any, kEdsTypeUrlUpb); + google_protobuf_Any_set_value(endpoint_any, + upb_strview_make(bytes, length)); + } + if (meta->client_status == XdsApi::ResourceMetadata::NACKED) { + // Set error_state if NACKED + envoy_admin_v3_EndpointsConfigDump_DynamicEndpointConfig_set_error_state( + dynamic_endpoint, CreateUpdateFailureStateUpb(context, meta)); + } + } +} + +} // namespace + +std::string XdsApi::AssembleClientConfig( + const ResourceTypeMetadataMap& resource_type_metadata_map) { + upb::Arena arena; + // Create the ClientConfig for resource metadata from XdsClient + auto* client_config = envoy_service_status_v3_ClientConfig_new(arena.ptr()); + // Fill-in the node information + auto* node = envoy_service_status_v3_ClientConfig_mutable_node(client_config, + arena.ptr()); + const EncodingContext context = {client_, tracer_, symtab_.ptr(), arena.ptr(), + true}; + PopulateNode(context, node_, build_version_, user_agent_name_, node); + // Dump each xDS-type config into PerXdsConfig + for (auto& p : resource_type_metadata_map) { + absl::string_view type_url = p.first; + const ResourceTypeMetadata& resource_type_metadata = p.second; + if (type_url == kLdsTypeUrl) { + auto* per_xds_config = + envoy_service_status_v3_ClientConfig_add_xds_config(client_config, + context.arena); + DumpLdsConfig(context, resource_type_metadata, per_xds_config); + } else if (type_url == kRdsTypeUrl) { + auto* per_xds_config = + envoy_service_status_v3_ClientConfig_add_xds_config(client_config, + context.arena); + DumpRdsConfig(context, resource_type_metadata, per_xds_config); + } else if (type_url == kCdsTypeUrl) { + auto* per_xds_config = + envoy_service_status_v3_ClientConfig_add_xds_config(client_config, + context.arena); + DumpCdsConfig(context, resource_type_metadata, per_xds_config); + } else if (type_url == kEdsTypeUrl) { + auto* per_xds_config = + envoy_service_status_v3_ClientConfig_add_xds_config(client_config, + context.arena); + DumpEdsConfig(context, resource_type_metadata, per_xds_config); + } else { + gpr_log(GPR_ERROR, "invalid type_url %s", std::string(type_url).c_str()); + return ""; + } + } + // Serialize the upb message to bytes + size_t output_length; + char* output = envoy_service_status_v3_ClientConfig_serialize( + client_config, arena.ptr(), &output_length); + return std::string(output, output_length); +} + } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_api.h b/grpc/src/core/ext/xds/xds_api.h index f2a67072..e7bf1cd1 100644 --- a/grpc/src/core/ext/xds/xds_api.h +++ b/grpc/src/core/ext/xds/xds_api.h @@ -33,9 +33,12 @@ #include <grpc/slice_buffer.h> +#include "envoy/admin/v3/config_dump.upb.h" #include "src/core/ext/filters/client_channel/server_address.h" #include "src/core/ext/xds/xds_bootstrap.h" #include "src/core/ext/xds/xds_client_stats.h" +#include "src/core/ext/xds/xds_http_filters.h" +#include "src/core/lib/matchers/matchers.h" namespace grpc_core { @@ -57,76 +60,58 @@ class XdsApi { int64_t seconds = 0; int32_t nanos = 0; bool operator==(const Duration& other) const { - return (seconds == other.seconds && nanos == other.nanos); + return seconds == other.seconds && nanos == other.nanos; } std::string ToString() const { return absl::StrFormat("Duration seconds: %ld, nanos %d", seconds, nanos); } }; + using TypedPerFilterConfig = + std::map<std::string, XdsHttpFilterImpl::FilterConfig>; + // TODO(donnadionne): When we can use absl::variant<>, consider using that // for: PathMatcher, HeaderMatcher, cluster_name and weighted_clusters struct Route { // Matchers for this route. struct Matchers { - struct PathMatcher { - enum class PathMatcherType { - PATH, // path stored in string_matcher field - PREFIX, // prefix stored in string_matcher field - REGEX, // regex stored in regex_matcher field - }; - PathMatcherType type; - std::string string_matcher; - std::unique_ptr<RE2> regex_matcher; - bool case_sensitive = true; - - PathMatcher() = default; - PathMatcher(const PathMatcher& other); - PathMatcher& operator=(const PathMatcher& other); - bool operator==(const PathMatcher& other) const; - std::string ToString() const; - }; - - struct HeaderMatcher { - enum class HeaderMatcherType { - EXACT, // value stored in string_matcher field - REGEX, // uses regex_match field - RANGE, // uses range_start and range_end fields - PRESENT, // uses present_match field - PREFIX, // prefix stored in string_matcher field - SUFFIX, // suffix stored in string_matcher field - }; - std::string name; - HeaderMatcherType type; - int64_t range_start; - int64_t range_end; - std::string string_matcher; - std::unique_ptr<RE2> regex_match; - bool present_match; - // invert_match field may or may not exisit, so initialize it to - // false. - bool invert_match = false; - - HeaderMatcher() = default; - HeaderMatcher(const HeaderMatcher& other); - HeaderMatcher& operator=(const HeaderMatcher& other); - bool operator==(const HeaderMatcher& other) const; - std::string ToString() const; - }; - - PathMatcher path_matcher; + StringMatcher path_matcher; std::vector<HeaderMatcher> header_matchers; absl::optional<uint32_t> fraction_per_million; bool operator==(const Matchers& other) const { - return (path_matcher == other.path_matcher && - header_matchers == other.header_matchers && - fraction_per_million == other.fraction_per_million); + return path_matcher == other.path_matcher && + header_matchers == other.header_matchers && + fraction_per_million == other.fraction_per_million; } std::string ToString() const; }; + struct HashPolicy { + enum Type { HEADER, CHANNEL_ID }; + Type type; + bool terminal = false; + // Fields used for type HEADER. + std::string header_name; + std::unique_ptr<RE2> regex = nullptr; + std::string regex_substitution; + + HashPolicy() {} + + // Copyable. + HashPolicy(const HashPolicy& other); + HashPolicy& operator=(const HashPolicy& other); + + // Moveable. + HashPolicy(HashPolicy&& other) noexcept; + HashPolicy& operator=(HashPolicy&& other) noexcept; + + bool operator==(const HashPolicy& other) const; + std::string ToString() const; + }; + Matchers matchers; + std::vector<HashPolicy> hash_policies; // Action for this route. // TODO(roth): When we can use absl::variant<>, consider using that @@ -135,8 +120,11 @@ class XdsApi { struct ClusterWeight { std::string name; uint32_t weight; + TypedPerFilterConfig typed_per_filter_config; + bool operator==(const ClusterWeight& other) const { - return (name == other.name && weight == other.weight); + return name == other.name && weight == other.weight && + typed_per_filter_config == other.typed_per_filter_config; } std::string ToString() const; }; @@ -147,11 +135,13 @@ class XdsApi { // not set. absl::optional<Duration> max_stream_duration; + TypedPerFilterConfig typed_per_filter_config; + bool operator==(const Route& other) const { - return (matchers == other.matchers && - cluster_name == other.cluster_name && - weighted_clusters == other.weighted_clusters && - max_stream_duration == other.max_stream_duration); + return matchers == other.matchers && cluster_name == other.cluster_name && + weighted_clusters == other.weighted_clusters && + max_stream_duration == other.max_stream_duration && + typed_per_filter_config == other.typed_per_filter_config; } std::string ToString() const; }; @@ -160,9 +150,11 @@ class XdsApi { struct VirtualHost { std::vector<std::string> domains; std::vector<Route> routes; + TypedPerFilterConfig typed_per_filter_config; bool operator==(const VirtualHost& other) const { - return domains == other.domains && routes == other.routes; + return domains == other.domains && routes == other.routes && + typed_per_filter_config == other.typed_per_filter_config; } }; @@ -175,42 +167,6 @@ class XdsApi { VirtualHost* FindVirtualHostForDomain(const std::string& domain); }; - class StringMatcher { - public: - enum class StringMatcherType { - EXACT, // value stored in string_matcher_ field - PREFIX, // value stored in string_matcher_ field - SUFFIX, // value stored in string_matcher_ field - SAFE_REGEX, // pattern stored in regex_matcher_ field - CONTAINS, // value stored in string_matcher_ field - }; - - StringMatcher() = default; - StringMatcher(const StringMatcher& other); - StringMatcher(StringMatcherType type, const std::string& matcher, - bool ignore_case = false); - StringMatcher& operator=(const StringMatcher& other); - bool operator==(const StringMatcher& other) const; - - bool Match(absl::string_view value) const; - - std::string ToString() const; - - StringMatcherType type() const { return type_; } - - // Valid for EXACT, PREFIX, SUFFIX and CONTAINS - const std::string& string_matcher() const { return string_matcher_; } - - // Valid for SAFE_REGEX - RE2* regex_matcher() const { return regex_matcher_.get(); } - - private: - StringMatcherType type_ = StringMatcherType::EXACT; - std::string string_matcher_; - std::unique_ptr<RE2> regex_matcher_; - bool ignore_case_ = false; - }; - struct CommonTlsContext { struct CertificateValidationContext { std::vector<StringMatcher> match_subject_alt_names; @@ -264,30 +220,182 @@ class XdsApi { bool Empty() const; }; + struct DownstreamTlsContext { + CommonTlsContext common_tls_context; + bool require_client_certificate = false; + + bool operator==(const DownstreamTlsContext& other) const { + return common_tls_context == other.common_tls_context && + require_client_certificate == other.require_client_certificate; + } + + std::string ToString() const; + bool Empty() const; + }; + // TODO(roth): When we can use absl::variant<>, consider using that // here, to enforce the fact that only one of the two fields can be set. struct LdsUpdate { - // The name to use in the RDS request. - std::string route_config_name; - // Storing the Http Connection Manager Common Http Protocol Option - // max_stream_duration - Duration http_max_stream_duration; - // The RouteConfiguration to use for this listener. - // Present only if it is inlined in the LDS response. - absl::optional<RdsUpdate> rds_update; + enum class ListenerType { + kTcpListener = 0, + kHttpApiListener, + } type; + + struct HttpConnectionManager { + // The name to use in the RDS request. + std::string route_config_name; + // Storing the Http Connection Manager Common Http Protocol Option + // max_stream_duration + Duration http_max_stream_duration; + // The RouteConfiguration to use for this listener. + // Present only if it is inlined in the LDS response. + absl::optional<RdsUpdate> rds_update; + + struct HttpFilter { + std::string name; + XdsHttpFilterImpl::FilterConfig config; + + bool operator==(const HttpFilter& other) const { + return name == other.name && config == other.config; + } + + std::string ToString() const; + }; + std::vector<HttpFilter> http_filters; + + bool operator==(const HttpConnectionManager& other) const { + return route_config_name == other.route_config_name && + http_max_stream_duration == other.http_max_stream_duration && + rds_update == other.rds_update && + http_filters == other.http_filters; + } + + std::string ToString() const; + }; + + // Populated for type=kHttpApiListener. + HttpConnectionManager http_connection_manager; + + // Populated for type=kTcpListener. + // host:port listening_address set when type is kTcpListener + std::string address; + + struct FilterChainData { + DownstreamTlsContext downstream_tls_context; + // This is in principle the filter list. + // We currently require exactly one filter, which is the HCM. + HttpConnectionManager http_connection_manager; + + bool operator==(const FilterChainData& other) const { + return downstream_tls_context == other.downstream_tls_context && + http_connection_manager == other.http_connection_manager; + } + + std::string ToString() const; + } filter_chain_data; + + // A multi-level map used to determine which filter chain to use for a given + // incoming connection. Determining the right filter chain for a given + // connection checks the following properties, in order: + // - destination port (never matched, so not present in map) + // - destination IP address + // - server name (never matched, so not present in map) + // - transport protocol (allows only "raw_buffer" or unset, prefers the + // former, so only one of those two types is present in map) + // - application protocol (never matched, so not present in map) + // - connection source type (any, local or external) + // - source IP address + // - source port + // https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/listener/v3/listener_components.proto#config-listener-v3-filterchainmatch + // for more details + struct FilterChainMap { + struct FilterChainDataSharedPtr { + std::shared_ptr<FilterChainData> data; + bool operator==(const FilterChainDataSharedPtr& other) const { + return *data == *other.data; + } + }; + struct CidrRange { + grpc_resolved_address address; + uint32_t prefix_len; + + bool operator==(const CidrRange& other) const { + return memcmp(&address, &other.address, sizeof(address)) == 0 && + prefix_len == other.prefix_len; + } + + std::string ToString() const; + }; + using SourcePortsMap = std::map<uint16_t, FilterChainDataSharedPtr>; + struct SourceIp { + absl::optional<CidrRange> prefix_range; + SourcePortsMap ports_map; + + bool operator==(const SourceIp& other) const { + return prefix_range == other.prefix_range && + ports_map == other.ports_map; + } + }; + using SourceIpVector = std::vector<SourceIp>; + enum class ConnectionSourceType { + kAny = 0, + kSameIpOrLoopback, + kExternal + }; + using ConnectionSourceTypesArray = std::array<SourceIpVector, 3>; + struct DestinationIp { + absl::optional<CidrRange> prefix_range; + // We always fail match on server name, so those filter chains are not + // included here. + ConnectionSourceTypesArray source_types_array; + + bool operator==(const DestinationIp& other) const { + return prefix_range == other.prefix_range && + source_types_array == other.source_types_array; + } + }; + // We always fail match on destination ports map + using DestinationIpVector = std::vector<DestinationIp>; + DestinationIpVector destination_ip_vector; + + bool operator==(const FilterChainMap& other) const { + return destination_ip_vector == other.destination_ip_vector; + } + + std::string ToString() const; + } filter_chain_map; + + absl::optional<FilterChainData> default_filter_chain; bool operator==(const LdsUpdate& other) const { - return route_config_name == other.route_config_name && - rds_update == other.rds_update && - http_max_stream_duration == other.http_max_stream_duration; + return http_connection_manager == other.http_connection_manager && + address == other.address && + filter_chain_map == other.filter_chain_map && + default_filter_chain == other.default_filter_chain; } + + std::string ToString() const; }; - using LdsUpdateMap = std::map<std::string /*server_name*/, LdsUpdate>; + struct LdsResourceData { + LdsUpdate resource; + std::string serialized_proto; + }; - using RdsUpdateMap = std::map<std::string /*route_config_name*/, RdsUpdate>; + using LdsUpdateMap = std::map<std::string /*server_name*/, LdsResourceData>; + + struct RdsResourceData { + RdsUpdate resource; + std::string serialized_proto; + }; + + using RdsUpdateMap = + std::map<std::string /*route_config_name*/, RdsResourceData>; struct CdsUpdate { + enum ClusterType { EDS, LOGICAL_DNS, AGGREGATE }; + ClusterType cluster_type; + // For cluster type EDS. // The name to use in the EDS request. // If empty, the cluster name will be used. std::string eds_service_name; @@ -298,22 +406,39 @@ class XdsApi { // If set to the empty string, will use the same server we obtained the CDS // data from. absl::optional<std::string> lrs_load_reporting_server_name; + // The LB policy to use (e.g., "ROUND_ROBIN" or "RING_HASH"). + std::string lb_policy; + // Used for RING_HASH LB policy only. + uint64_t min_ring_size = 1024; + uint64_t max_ring_size = 8388608; + enum HashFunction { XX_HASH, MURMUR_HASH_2 }; + HashFunction hash_function; // Maximum number of outstanding requests can be made to the upstream // cluster. uint32_t max_concurrent_requests = 1024; + // For cluster type AGGREGATE. + // The prioritized list of cluster names. + std::vector<std::string> prioritized_cluster_names; bool operator==(const CdsUpdate& other) const { - return eds_service_name == other.eds_service_name && + return cluster_type == other.cluster_type && + eds_service_name == other.eds_service_name && common_tls_context == other.common_tls_context && lrs_load_reporting_server_name == other.lrs_load_reporting_server_name && + prioritized_cluster_names == other.prioritized_cluster_names && max_concurrent_requests == other.max_concurrent_requests; } std::string ToString() const; }; - using CdsUpdateMap = std::map<std::string /*cluster_name*/, CdsUpdate>; + struct CdsResourceData { + CdsUpdate resource; + std::string serialized_proto; + }; + + using CdsUpdateMap = std::map<std::string /*cluster_name*/, CdsResourceData>; struct EdsUpdate { struct Priority { @@ -397,7 +522,13 @@ class XdsApi { std::string ToString() const; }; - using EdsUpdateMap = std::map<std::string /*eds_service_name*/, EdsUpdate>; + struct EdsResourceData { + EdsUpdate resource; + std::string serialized_proto; + }; + + using EdsUpdateMap = + std::map<std::string /*eds_service_name*/, EdsResourceData>; struct ClusterLoadReport { XdsClusterDropStats::Snapshot dropped_requests; @@ -410,22 +541,75 @@ class XdsApi { std::pair<std::string /*cluster_name*/, std::string /*eds_service_name*/>, ClusterLoadReport>; - XdsApi(XdsClient* client, TraceFlag* tracer, const XdsBootstrap::Node* node); + // The metadata of the xDS resource; used by the xDS config dump. + struct ResourceMetadata { + // Resource status from the view of a xDS client, which tells the + // synchronization status between the xDS client and the xDS server. + enum ClientResourceStatus { + // Client requested this resource but hasn't received any update from + // management server. The client will not fail requests, but will queue + // them + // until update arrives or the client times out waiting for the resource. + REQUESTED = 1, + // This resource has been requested by the client but has either not been + // delivered by the server or was previously delivered by the server and + // then subsequently removed from resources provided by the server. + DOES_NOT_EXIST, + // Client received this resource and replied with ACK. + ACKED, + // Client received this resource and replied with NACK. + NACKED + }; - // Creates an ADS request. - // Takes ownership of \a error. - grpc_slice CreateAdsRequest(const XdsBootstrap::XdsServer& server, - const std::string& type_url, - const std::set<absl::string_view>& resource_names, - const std::string& version, - const std::string& nonce, grpc_error* error, - bool populate_node); + // The client status of this resource. + ClientResourceStatus client_status = REQUESTED; + // The serialized bytes of the last successfully updated raw xDS resource. + std::string serialized_proto; + // The timestamp when the resource was last successfully updated. + grpc_millis update_time = 0; + // The last successfully updated version of the resource. + std::string version; + // The rejected version string of the last failed update attempt. + std::string failed_version; + // Details about the last failed update attempt. + std::string failed_details; + // Timestamp of the last failed update attempt. + grpc_millis failed_update_time = 0; + }; + using ResourceMetadataMap = + std::map<absl::string_view /*resource_name*/, const ResourceMetadata*>; + struct ResourceTypeMetadata { + absl::string_view version; + ResourceMetadataMap resource_metadata_map; + }; + using ResourceTypeMetadataMap = + std::map<absl::string_view /*type_url*/, ResourceTypeMetadata>; + static_assert(static_cast<ResourceMetadata::ClientResourceStatus>( + envoy_admin_v3_REQUESTED) == + ResourceMetadata::ClientResourceStatus::REQUESTED, + ""); + static_assert(static_cast<ResourceMetadata::ClientResourceStatus>( + envoy_admin_v3_DOES_NOT_EXIST) == + ResourceMetadata::ClientResourceStatus::DOES_NOT_EXIST, + ""); + static_assert(static_cast<ResourceMetadata::ClientResourceStatus>( + envoy_admin_v3_ACKED) == + ResourceMetadata::ClientResourceStatus::ACKED, + ""); + static_assert(static_cast<ResourceMetadata::ClientResourceStatus>( + envoy_admin_v3_NACKED) == + ResourceMetadata::ClientResourceStatus::NACKED, + ""); - // Parses an ADS response. // If the response can't be parsed at the top level, the resulting // type_url will be empty. + // If there is any other type of validation error, the parse_error + // field will be set to something other than GRPC_ERROR_NONE and the + // resource_names_failed field will be populated. + // Otherwise, one of the *_update_map fields will be populated, based + // on the type_url field. struct AdsParseResult { - grpc_error* parse_error = GRPC_ERROR_NONE; + grpc_error_handle parse_error = GRPC_ERROR_NONE; std::string version; std::string nonce; std::string type_url; @@ -433,9 +617,23 @@ class XdsApi { RdsUpdateMap rds_update_map; CdsUpdateMap cds_update_map; EdsUpdateMap eds_update_map; + std::set<std::string> resource_names_failed; }; + + XdsApi(XdsClient* client, TraceFlag* tracer, const XdsBootstrap::Node* node); + + // Creates an ADS request. + // Takes ownership of \a error. + grpc_slice CreateAdsRequest(const XdsBootstrap::XdsServer& server, + const std::string& type_url, + const std::set<absl::string_view>& resource_names, + const std::string& version, + const std::string& nonce, grpc_error_handle error, + bool populate_node); + + // Parses an ADS response. AdsParseResult ParseAdsResponse( - const grpc_slice& encoded_response, + const XdsBootstrap::XdsServer& server, const grpc_slice& encoded_response, const std::set<absl::string_view>& expected_listener_names, const std::set<absl::string_view>& expected_route_configuration_names, const std::set<absl::string_view>& expected_cluster_names, @@ -450,10 +648,14 @@ class XdsApi { // Parses the LRS response and returns \a // load_reporting_interval for client-side load reporting. If there is any // error, the output config is invalid. - grpc_error* ParseLrsResponse(const grpc_slice& encoded_response, - bool* send_all_clusters, - std::set<std::string>* cluster_names, - grpc_millis* load_reporting_interval); + grpc_error_handle ParseLrsResponse(const grpc_slice& encoded_response, + bool* send_all_clusters, + std::set<std::string>* cluster_names, + grpc_millis* load_reporting_interval); + + // Assemble the client config proto message and return the serialized result. + std::string AssembleClientConfig( + const ResourceTypeMetadataMap& resource_type_metadata_map); private: XdsClient* client_; diff --git a/grpc/src/core/ext/xds/xds_bootstrap.cc b/grpc/src/core/ext/xds/xds_bootstrap.cc index e48d9827..33cf276d 100644 --- a/grpc/src/core/ext/xds/xds_bootstrap.cc +++ b/grpc/src/core/ext/xds/xds_bootstrap.cc @@ -30,7 +30,6 @@ #include "src/core/ext/xds/certificate_provider_registry.h" #include "src/core/ext/xds/xds_api.h" -#include "src/core/lib/gpr/env.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/iomgr/load_file.h" #include "src/core/lib/security/credentials/credentials.h" @@ -48,8 +47,8 @@ bool XdsChannelCredsRegistry::IsSupported(const std::string& creds_type) { creds_type == "fake"; } -bool XdsChannelCredsRegistry::IsValidConfig(const std::string& creds_type, - const Json& config) { +bool XdsChannelCredsRegistry::IsValidConfig(const std::string& /*creds_type*/, + const Json& /*config*/) { // Currently, none of the creds types actually take a config, but we // ignore whatever might be specified in the bootstrap file for // forward compatibility reasons. @@ -58,7 +57,7 @@ bool XdsChannelCredsRegistry::IsValidConfig(const std::string& creds_type, RefCountedPtr<grpc_channel_credentials> XdsChannelCredsRegistry::MakeChannelCreds(const std::string& creds_type, - const Json& config) { + const Json& /*config*/) { if (creds_type == "google_default") { return grpc_google_default_credentials_create(nullptr); } else if (creds_type == "insecure") { @@ -81,109 +80,27 @@ bool XdsBootstrap::XdsServer::ShouldUseV3() const { // XdsBootstrap // -namespace { - -std::string BootstrapString(const XdsBootstrap& bootstrap) { - std::vector<std::string> parts; - if (bootstrap.node() != nullptr) { - parts.push_back(absl::StrFormat( - "node={\n" - " id=\"%s\",\n" - " cluster=\"%s\",\n" - " locality={\n" - " region=\"%s\",\n" - " zone=\"%s\",\n" - " subzone=\"%s\"\n" - " },\n" - " metadata=%s,\n" - "},\n", - bootstrap.node()->id, bootstrap.node()->cluster, - bootstrap.node()->locality_region, bootstrap.node()->locality_zone, - bootstrap.node()->locality_subzone, bootstrap.node()->metadata.Dump())); - } - parts.push_back(absl::StrFormat( - "servers=[\n" - " {\n" - " uri=\"%s\",\n" - " creds_type=%s,\n", - bootstrap.server().server_uri, bootstrap.server().channel_creds_type)); - if (bootstrap.server().channel_creds_config.type() != Json::Type::JSON_NULL) { - parts.push_back( - absl::StrFormat(" creds_config=%s,", - bootstrap.server().channel_creds_config.Dump())); - } - if (!bootstrap.server().server_features.empty()) { - parts.push_back(absl::StrCat( - " server_features=[", - absl::StrJoin(bootstrap.server().server_features, ", "), "],\n")); - } - parts.push_back(" }\n],\n"); - parts.push_back("certificate_providers={\n"); - for (const auto& entry : bootstrap.certificate_providers()) { - parts.push_back( - absl::StrFormat(" %s={\n" - " plugin_name=%s\n" - " config=%s\n" - " },\n", - entry.first, entry.second.plugin_name, - entry.second.config->ToString())); - } - parts.push_back("}"); - return absl::StrJoin(parts, ""); -} - -} // namespace - -std::unique_ptr<XdsBootstrap> XdsBootstrap::ReadFromFile(XdsClient* client, - TraceFlag* tracer, - grpc_error** error) { - grpc_core::UniquePtr<char> path(gpr_getenv("GRPC_XDS_BOOTSTRAP")); - if (path == nullptr) { - *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Environment variable GRPC_XDS_BOOTSTRAP not defined"); - return nullptr; - } - if (GRPC_TRACE_FLAG_ENABLED(*tracer)) { - gpr_log(GPR_INFO, - "[xds_client %p] Got bootstrap file location from " - "GRPC_XDS_BOOTSTRAP environment variable: %s", - client, path.get()); - } - grpc_slice contents; - *error = grpc_load_file(path.get(), /*add_null_terminator=*/true, &contents); - if (*error != GRPC_ERROR_NONE) return nullptr; - absl::string_view contents_str_view = StringViewFromSlice(contents); - if (GRPC_TRACE_FLAG_ENABLED(*tracer)) { - gpr_log(GPR_DEBUG, "[xds_client %p] Bootstrap file contents: %s", client, - std::string(contents_str_view).c_str()); - } - Json json = Json::Parse(contents_str_view, error); - grpc_slice_unref_internal(contents); +std::unique_ptr<XdsBootstrap> XdsBootstrap::Create( + absl::string_view json_string, grpc_error_handle* error) { + Json json = Json::Parse(json_string, error); if (*error != GRPC_ERROR_NONE) { - grpc_error* error_out = GRPC_ERROR_CREATE_REFERENCING_FROM_COPIED_STRING( - absl::StrCat("Failed to parse bootstrap file ", path.get()).c_str(), - error, 1); + grpc_error_handle error_out = + GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( + "Failed to parse bootstrap JSON string", error, 1); GRPC_ERROR_UNREF(*error); *error = error_out; return nullptr; } - std::unique_ptr<XdsBootstrap> result = - absl::make_unique<XdsBootstrap>(std::move(json), error); - if (*error == GRPC_ERROR_NONE && GRPC_TRACE_FLAG_ENABLED(*tracer)) { - gpr_log(GPR_INFO, - "[xds_client %p] Bootstrap config for creating xds client:\n%s", - client, BootstrapString(*result).c_str()); - } - return result; + return absl::make_unique<XdsBootstrap>(std::move(json), error); } -XdsBootstrap::XdsBootstrap(Json json, grpc_error** error) { +XdsBootstrap::XdsBootstrap(Json json, grpc_error_handle* error) { if (json.type() != Json::Type::OBJECT) { *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( "malformed JSON in bootstrap file"); return; } - std::vector<grpc_error*> error_list; + std::vector<grpc_error_handle> error_list; auto it = json.mutable_object()->find("xds_servers"); if (it == json.mutable_object()->end()) { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( @@ -192,7 +109,7 @@ XdsBootstrap::XdsBootstrap(Json json, grpc_error** error) { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "\"xds_servers\" field is not an array")); } else { - grpc_error* parse_error = ParseXdsServerList(&it->second); + grpc_error_handle parse_error = ParseXdsServerList(&it->second); if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); } it = json.mutable_object()->find("node"); @@ -201,10 +118,20 @@ XdsBootstrap::XdsBootstrap(Json json, grpc_error** error) { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "\"node\" field is not an object")); } else { - grpc_error* parse_error = ParseNode(&it->second); + grpc_error_handle parse_error = ParseNode(&it->second); if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); } } + it = json.mutable_object()->find("server_listener_resource_name_template"); + if (it != json.mutable_object()->end()) { + if (it->second.type() != Json::Type::STRING) { + error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "\"server_listener_resource_name_template\" field is not a string")); + } else { + server_listener_resource_name_template_ = + std::move(*it->second.mutable_string_value()); + } + } if (XdsSecurityEnabled()) { it = json.mutable_object()->find("certificate_providers"); if (it != json.mutable_object()->end()) { @@ -212,7 +139,7 @@ XdsBootstrap::XdsBootstrap(Json json, grpc_error** error) { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "\"certificate_providers\" field is not an object")); } else { - grpc_error* parse_error = ParseCertificateProviders(&it->second); + grpc_error_handle parse_error = ParseCertificateProviders(&it->second); if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); } } @@ -221,15 +148,15 @@ XdsBootstrap::XdsBootstrap(Json json, grpc_error** error) { &error_list); } -grpc_error* XdsBootstrap::ParseXdsServerList(Json* json) { - std::vector<grpc_error*> error_list; +grpc_error_handle XdsBootstrap::ParseXdsServerList(Json* json) { + std::vector<grpc_error_handle> error_list; for (size_t i = 0; i < json->mutable_array()->size(); ++i) { Json& child = json->mutable_array()->at(i); if (child.type() != Json::Type::OBJECT) { error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrCat("array element ", i, " is not an object").c_str())); } else { - grpc_error* parse_error = ParseXdsServer(&child, i); + grpc_error_handle parse_error = ParseXdsServer(&child, i); if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); } } @@ -237,8 +164,8 @@ grpc_error* XdsBootstrap::ParseXdsServerList(Json* json) { &error_list); } -grpc_error* XdsBootstrap::ParseXdsServer(Json* json, size_t idx) { - std::vector<grpc_error*> error_list; +grpc_error_handle XdsBootstrap::ParseXdsServer(Json* json, size_t idx) { + std::vector<grpc_error_handle> error_list; servers_.emplace_back(); XdsServer& server = servers_[servers_.size() - 1]; auto it = json->mutable_object()->find("server_uri"); @@ -259,7 +186,8 @@ grpc_error* XdsBootstrap::ParseXdsServer(Json* json, size_t idx) { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "\"channel_creds\" field is not an array")); } else { - grpc_error* parse_error = ParseChannelCredsArray(&it->second, &server); + grpc_error_handle parse_error = + ParseChannelCredsArray(&it->second, &server); if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); } it = json->mutable_object()->find("server_features"); @@ -268,14 +196,15 @@ grpc_error* XdsBootstrap::ParseXdsServer(Json* json, size_t idx) { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "\"server_features\" field is not an array")); } else { - grpc_error* parse_error = ParseServerFeaturesArray(&it->second, &server); + grpc_error_handle parse_error = + ParseServerFeaturesArray(&it->second, &server); if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); } } // Can't use GRPC_ERROR_CREATE_FROM_VECTOR() here, because the error // string is not static in this case. if (error_list.empty()) return GRPC_ERROR_NONE; - grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( + grpc_error_handle error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrCat("errors parsing index ", idx).c_str()); for (size_t i = 0; i < error_list.size(); ++i) { error = grpc_error_add_child(error, error_list[i]); @@ -283,16 +212,16 @@ grpc_error* XdsBootstrap::ParseXdsServer(Json* json, size_t idx) { return error; } -grpc_error* XdsBootstrap::ParseChannelCredsArray(Json* json, - XdsServer* server) { - std::vector<grpc_error*> error_list; +grpc_error_handle XdsBootstrap::ParseChannelCredsArray(Json* json, + XdsServer* server) { + std::vector<grpc_error_handle> error_list; for (size_t i = 0; i < json->mutable_array()->size(); ++i) { Json& child = json->mutable_array()->at(i); if (child.type() != Json::Type::OBJECT) { error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrCat("array element ", i, " is not an object").c_str())); } else { - grpc_error* parse_error = ParseChannelCreds(&child, i, server); + grpc_error_handle parse_error = ParseChannelCreds(&child, i, server); if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); } } @@ -304,9 +233,9 @@ grpc_error* XdsBootstrap::ParseChannelCredsArray(Json* json, &error_list); } -grpc_error* XdsBootstrap::ParseChannelCreds(Json* json, size_t idx, - XdsServer* server) { - std::vector<grpc_error*> error_list; +grpc_error_handle XdsBootstrap::ParseChannelCreds(Json* json, size_t idx, + XdsServer* server) { + std::vector<grpc_error_handle> error_list; std::string type; auto it = json->mutable_object()->find("type"); if (it == json->mutable_object()->end()) { @@ -342,7 +271,7 @@ grpc_error* XdsBootstrap::ParseChannelCreds(Json* json, size_t idx, // Can't use GRPC_ERROR_CREATE_FROM_VECTOR() here, because the error // string is not static in this case. if (error_list.empty()) return GRPC_ERROR_NONE; - grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( + grpc_error_handle error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrCat("errors parsing index ", idx).c_str()); for (size_t i = 0; i < error_list.size(); ++i) { error = grpc_error_add_child(error, error_list[i]); @@ -350,30 +279,22 @@ grpc_error* XdsBootstrap::ParseChannelCreds(Json* json, size_t idx, return error; } -grpc_error* XdsBootstrap::ParseServerFeaturesArray(Json* json, - XdsServer* server) { - std::vector<grpc_error*> error_list; +grpc_error_handle XdsBootstrap::ParseServerFeaturesArray(Json* json, + XdsServer* server) { + std::vector<grpc_error_handle> error_list; for (size_t i = 0; i < json->mutable_array()->size(); ++i) { Json& child = json->mutable_array()->at(i); if (child.type() == Json::Type::STRING && child.string_value() == "xds_v3") { - // TODO(roth): Remove env var check once we do interop testing and - // are sure that the v3 code actually works. - grpc_core::UniquePtr<char> enable_str( - gpr_getenv("GRPC_XDS_EXPERIMENTAL_V3_SUPPORT")); - bool enabled = false; - if (gpr_parse_bool_value(enable_str.get(), &enabled) && enabled) { - server->server_features.insert( - std::move(*child.mutable_string_value())); - } + server->server_features.insert(std::move(*child.mutable_string_value())); } } return GRPC_ERROR_CREATE_FROM_VECTOR( "errors parsing \"server_features\" array", &error_list); } -grpc_error* XdsBootstrap::ParseNode(Json* json) { - std::vector<grpc_error*> error_list; +grpc_error_handle XdsBootstrap::ParseNode(Json* json) { + std::vector<grpc_error_handle> error_list; node_ = absl::make_unique<Node>(); auto it = json->mutable_object()->find("id"); if (it != json->mutable_object()->end()) { @@ -399,7 +320,7 @@ grpc_error* XdsBootstrap::ParseNode(Json* json) { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "\"locality\" field is not an object")); } else { - grpc_error* parse_error = ParseLocality(&it->second); + grpc_error_handle parse_error = ParseLocality(&it->second); if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); } } @@ -416,8 +337,8 @@ grpc_error* XdsBootstrap::ParseNode(Json* json) { &error_list); } -grpc_error* XdsBootstrap::ParseLocality(Json* json) { - std::vector<grpc_error*> error_list; +grpc_error_handle XdsBootstrap::ParseLocality(Json* json) { + std::vector<grpc_error_handle> error_list; auto it = json->mutable_object()->find("region"); if (it != json->mutable_object()->end()) { if (it->second.type() != Json::Type::STRING) { @@ -436,21 +357,21 @@ grpc_error* XdsBootstrap::ParseLocality(Json* json) { node_->locality_zone = std::move(*it->second.mutable_string_value()); } } - it = json->mutable_object()->find("subzone"); + it = json->mutable_object()->find("sub_zone"); if (it != json->mutable_object()->end()) { if (it->second.type() != Json::Type::STRING) { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"subzone\" field is not a string")); + "\"sub_zone\" field is not a string")); } else { - node_->locality_subzone = std::move(*it->second.mutable_string_value()); + node_->locality_sub_zone = std::move(*it->second.mutable_string_value()); } } return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing \"locality\" object", &error_list); } -grpc_error* XdsBootstrap::ParseCertificateProviders(Json* json) { - std::vector<grpc_error*> error_list; +grpc_error_handle XdsBootstrap::ParseCertificateProviders(Json* json) { + std::vector<grpc_error_handle> error_list; for (auto& certificate_provider : *(json->mutable_object())) { if (certificate_provider.second.type() != Json::Type::OBJECT) { error_list.push_back(GRPC_ERROR_CREATE_FROM_COPIED_STRING( @@ -458,7 +379,7 @@ grpc_error* XdsBootstrap::ParseCertificateProviders(Json* json) { "\" is not an object") .c_str())); } else { - grpc_error* parse_error = ParseCertificateProvider( + grpc_error_handle parse_error = ParseCertificateProvider( certificate_provider.first, &certificate_provider.second); if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); } @@ -467,9 +388,9 @@ grpc_error* XdsBootstrap::ParseCertificateProviders(Json* json) { "errors parsing \"certificate_providers\" object", &error_list); } -grpc_error* XdsBootstrap::ParseCertificateProvider( +grpc_error_handle XdsBootstrap::ParseCertificateProvider( const std::string& instance_name, Json* certificate_provider_json) { - std::vector<grpc_error*> error_list; + std::vector<grpc_error_handle> error_list; auto it = certificate_provider_json->mutable_object()->find("plugin_name"); if (it == certificate_provider_json->mutable_object()->end()) { error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( @@ -490,14 +411,14 @@ grpc_error* XdsBootstrap::ParseCertificateProvider( error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( "\"config\" field is not an object")); } else { - grpc_error* parse_error = GRPC_ERROR_NONE; + grpc_error_handle parse_error = GRPC_ERROR_NONE; config = factory->CreateCertificateProviderConfig(it->second, &parse_error); if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); } } else { // "config" is an optional field, so create an empty JSON object. - grpc_error* parse_error = GRPC_ERROR_NONE; + grpc_error_handle parse_error = GRPC_ERROR_NONE; config = factory->CreateCertificateProviderConfig(Json::Object(), &parse_error); if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); @@ -509,7 +430,7 @@ grpc_error* XdsBootstrap::ParseCertificateProvider( // Can't use GRPC_ERROR_CREATE_FROM_VECTOR() here, because the error // string is not static in this case. if (error_list.empty()) return GRPC_ERROR_NONE; - grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( + grpc_error_handle error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrCat("errors parsing element \"", instance_name, "\"").c_str()); for (size_t i = 0; i < error_list.size(); ++i) { error = grpc_error_add_child(error, error_list[i]); @@ -517,4 +438,56 @@ grpc_error* XdsBootstrap::ParseCertificateProvider( return error; } +std::string XdsBootstrap::ToString() const { + std::vector<std::string> parts; + if (node_ != nullptr) { + parts.push_back(absl::StrFormat( + "node={\n" + " id=\"%s\",\n" + " cluster=\"%s\",\n" + " locality={\n" + " region=\"%s\",\n" + " zone=\"%s\",\n" + " sub_zone=\"%s\"\n" + " },\n" + " metadata=%s,\n" + "},\n", + node_->id, node_->cluster, node_->locality_region, node_->locality_zone, + node_->locality_sub_zone, node_->metadata.Dump())); + } + parts.push_back( + absl::StrFormat("servers=[\n" + " {\n" + " uri=\"%s\",\n" + " creds_type=%s,\n", + server().server_uri, server().channel_creds_type)); + if (server().channel_creds_config.type() != Json::Type::JSON_NULL) { + parts.push_back(absl::StrFormat(" creds_config=%s,", + server().channel_creds_config.Dump())); + } + if (!server().server_features.empty()) { + parts.push_back(absl::StrCat(" server_features=[", + absl::StrJoin(server().server_features, ", "), + "],\n")); + } + parts.push_back(" }\n],\n"); + if (!server_listener_resource_name_template_.empty()) { + parts.push_back( + absl::StrFormat("server_listener_resource_name_template=\"%s\",\n", + server_listener_resource_name_template_)); + } + parts.push_back("certificate_providers={\n"); + for (const auto& entry : certificate_providers_) { + parts.push_back( + absl::StrFormat(" %s={\n" + " plugin_name=%s\n" + " config=%s\n" + " },\n", + entry.first, entry.second.plugin_name, + entry.second.config->ToString())); + } + parts.push_back("}"); + return absl::StrJoin(parts, ""); +} + } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_bootstrap.h b/grpc/src/core/ext/xds/xds_bootstrap.h index 969d5d54..eff5e7ab 100644 --- a/grpc/src/core/ext/xds/xds_bootstrap.h +++ b/grpc/src/core/ext/xds/xds_bootstrap.h @@ -54,7 +54,7 @@ class XdsBootstrap { std::string cluster; std::string locality_region; std::string locality_zone; - std::string locality_subzone; + std::string locality_sub_zone; Json metadata; }; @@ -67,19 +67,24 @@ class XdsBootstrap { bool ShouldUseV3() const; }; + // Creates bootstrap object from json_string. // If *error is not GRPC_ERROR_NONE after returning, then there was an - // error reading the file. - static std::unique_ptr<XdsBootstrap> ReadFromFile(XdsClient* client, - TraceFlag* tracer, - grpc_error** error); + // error parsing the contents. + static std::unique_ptr<XdsBootstrap> Create(absl::string_view json_string, + grpc_error_handle* error); - // Do not instantiate directly -- use ReadFromFile() above instead. - XdsBootstrap(Json json, grpc_error** error); + // Do not instantiate directly -- use Create() above instead. + XdsBootstrap(Json json, grpc_error_handle* error); + + std::string ToString() const; // TODO(roth): We currently support only one server. Fix this when we // add support for fallback for the xds channel. const XdsServer& server() const { return servers_[0]; } const Node* node() const { return node_.get(); } + const std::string& server_listener_resource_name_template() const { + return server_listener_resource_name_template_; + } const CertificateProviderStore::PluginDefinitionMap& certificate_providers() const { @@ -87,19 +92,21 @@ class XdsBootstrap { } private: - grpc_error* ParseXdsServerList(Json* json); - grpc_error* ParseXdsServer(Json* json, size_t idx); - grpc_error* ParseChannelCredsArray(Json* json, XdsServer* server); - grpc_error* ParseChannelCreds(Json* json, size_t idx, XdsServer* server); - grpc_error* ParseServerFeaturesArray(Json* json, XdsServer* server); - grpc_error* ParseNode(Json* json); - grpc_error* ParseLocality(Json* json); - grpc_error* ParseCertificateProviders(Json* json); - grpc_error* ParseCertificateProvider(const std::string& instance_name, - Json* certificate_provider_json); + grpc_error_handle ParseXdsServerList(Json* json); + grpc_error_handle ParseXdsServer(Json* json, size_t idx); + grpc_error_handle ParseChannelCredsArray(Json* json, XdsServer* server); + grpc_error_handle ParseChannelCreds(Json* json, size_t idx, + XdsServer* server); + grpc_error_handle ParseServerFeaturesArray(Json* json, XdsServer* server); + grpc_error_handle ParseNode(Json* json); + grpc_error_handle ParseLocality(Json* json); + grpc_error_handle ParseCertificateProviders(Json* json); + grpc_error_handle ParseCertificateProvider(const std::string& instance_name, + Json* certificate_provider_json); absl::InlinedVector<XdsServer, 1> servers_; std::unique_ptr<Node> node_; + std::string server_listener_resource_name_template_; CertificateProviderStore::PluginDefinitionMap certificate_providers_; }; diff --git a/grpc/src/core/ext/xds/xds_certificate_provider.cc b/grpc/src/core/ext/xds/xds_certificate_provider.cc index f285b6db..ce1ba673 100644 --- a/grpc/src/core/ext/xds/xds_certificate_provider.cc +++ b/grpc/src/core/ext/xds/xds_certificate_provider.cc @@ -37,23 +37,24 @@ class RootCertificatesWatcher // presently, the watcher is immediately deleted when // CancelTlsCertificatesWatch() is called, but that can potentially change in // the future. - explicit RootCertificatesWatcher( - RefCountedPtr<grpc_tls_certificate_distributor> parent) - : parent_(std::move(parent)) {} + RootCertificatesWatcher( + RefCountedPtr<grpc_tls_certificate_distributor> parent, + std::string cert_name) + : parent_(std::move(parent)), cert_name_(std::move(cert_name)) {} void OnCertificatesChanged(absl::optional<absl::string_view> root_certs, absl::optional<PemKeyCertPairList> /* key_cert_pairs */) override { if (root_certs.has_value()) { - parent_->SetKeyMaterials("", std::string(root_certs.value()), + parent_->SetKeyMaterials(cert_name_, std::string(root_certs.value()), absl::nullopt); } } - void OnError(grpc_error* root_cert_error, - grpc_error* identity_cert_error) override { + void OnError(grpc_error_handle root_cert_error, + grpc_error_handle identity_cert_error) override { if (root_cert_error != GRPC_ERROR_NONE) { - parent_->SetErrorForCert("", root_cert_error /* pass the ref */, + parent_->SetErrorForCert(cert_name_, root_cert_error /* pass the ref */, absl::nullopt); } GRPC_ERROR_UNREF(identity_cert_error); @@ -61,6 +62,7 @@ class RootCertificatesWatcher private: RefCountedPtr<grpc_tls_certificate_distributor> parent_; + std::string cert_name_; }; class IdentityCertificatesWatcher @@ -71,22 +73,23 @@ class IdentityCertificatesWatcher // presently, the watcher is immediately deleted when // CancelTlsCertificatesWatch() is called, but that can potentially change in // the future. - explicit IdentityCertificatesWatcher( - RefCountedPtr<grpc_tls_certificate_distributor> parent) - : parent_(std::move(parent)) {} + IdentityCertificatesWatcher( + RefCountedPtr<grpc_tls_certificate_distributor> parent, + std::string cert_name) + : parent_(std::move(parent)), cert_name_(std::move(cert_name)) {} void OnCertificatesChanged( absl::optional<absl::string_view> /* root_certs */, absl::optional<PemKeyCertPairList> key_cert_pairs) override { if (key_cert_pairs.has_value()) { - parent_->SetKeyMaterials("", absl::nullopt, key_cert_pairs); + parent_->SetKeyMaterials(cert_name_, absl::nullopt, key_cert_pairs); } } - void OnError(grpc_error* root_cert_error, - grpc_error* identity_cert_error) override { + void OnError(grpc_error_handle root_cert_error, + grpc_error_handle identity_cert_error) override { if (identity_cert_error != GRPC_ERROR_NONE) { - parent_->SetErrorForCert("", absl::nullopt, + parent_->SetErrorForCert(cert_name_, absl::nullopt, identity_cert_error /* pass the ref */); } GRPC_ERROR_UNREF(root_cert_error); @@ -94,34 +97,35 @@ class IdentityCertificatesWatcher private: RefCountedPtr<grpc_tls_certificate_distributor> parent_; + std::string cert_name_; }; } // namespace -XdsCertificateProvider::XdsCertificateProvider( - absl::string_view root_cert_name, - RefCountedPtr<grpc_tls_certificate_distributor> root_cert_distributor, - absl::string_view identity_cert_name, - RefCountedPtr<grpc_tls_certificate_distributor> identity_cert_distributor, - std::vector<XdsApi::StringMatcher> san_matchers) - : root_cert_name_(root_cert_name), - identity_cert_name_(identity_cert_name), - root_cert_distributor_(std::move(root_cert_distributor)), - identity_cert_distributor_(std::move(identity_cert_distributor)), - san_matchers_(std::move(san_matchers)), - distributor_(MakeRefCounted<grpc_tls_certificate_distributor>()) { - distributor_->SetWatchStatusCallback( - absl::bind_front(&XdsCertificateProvider::WatchStatusCallback, this)); +// +// XdsCertificateProvider::ClusterCertificateState +// + +XdsCertificateProvider::ClusterCertificateState::~ClusterCertificateState() { + if (root_cert_watcher_ != nullptr) { + root_cert_distributor_->CancelTlsCertificatesWatch(root_cert_watcher_); + } + if (identity_cert_watcher_ != nullptr) { + identity_cert_distributor_->CancelTlsCertificatesWatch( + identity_cert_watcher_); + } } -XdsCertificateProvider::~XdsCertificateProvider() { - distributor_->SetWatchStatusCallback(nullptr); +bool XdsCertificateProvider::ClusterCertificateState::IsSafeToRemove() const { + return !watching_root_certs_ && !watching_identity_certs_ && + root_cert_distributor_ == nullptr && + identity_cert_distributor_ == nullptr; } -void XdsCertificateProvider::UpdateRootCertNameAndDistributor( - absl::string_view root_cert_name, - RefCountedPtr<grpc_tls_certificate_distributor> root_cert_distributor) { - MutexLock lock(&mu_); +void XdsCertificateProvider::ClusterCertificateState:: + UpdateRootCertNameAndDistributor( + const std::string& cert_name, absl::string_view root_cert_name, + RefCountedPtr<grpc_tls_certificate_distributor> root_cert_distributor) { if (root_cert_name_ == root_cert_name && root_cert_distributor_ == root_cert_distributor) { return; @@ -133,10 +137,10 @@ void XdsCertificateProvider::UpdateRootCertNameAndDistributor( root_cert_distributor_->CancelTlsCertificatesWatch(root_cert_watcher_); } if (root_cert_distributor != nullptr) { - UpdateRootCertWatcher(root_cert_distributor.get()); + UpdateRootCertWatcher(cert_name, root_cert_distributor.get()); } else { root_cert_watcher_ = nullptr; - distributor_->SetErrorForCert( + xds_certificate_provider_->distributor_->SetErrorForCert( "", GRPC_ERROR_CREATE_FROM_STATIC_STRING( "No certificate provider available for root certificates"), @@ -147,10 +151,11 @@ void XdsCertificateProvider::UpdateRootCertNameAndDistributor( root_cert_distributor_ = std::move(root_cert_distributor); } -void XdsCertificateProvider::UpdateIdentityCertNameAndDistributor( - absl::string_view identity_cert_name, - RefCountedPtr<grpc_tls_certificate_distributor> identity_cert_distributor) { - MutexLock lock(&mu_); +void XdsCertificateProvider::ClusterCertificateState:: + UpdateIdentityCertNameAndDistributor( + const std::string& cert_name, absl::string_view identity_cert_name, + RefCountedPtr<grpc_tls_certificate_distributor> + identity_cert_distributor) { if (identity_cert_name_ == identity_cert_name && identity_cert_distributor_ == identity_cert_distributor) { return; @@ -163,10 +168,10 @@ void XdsCertificateProvider::UpdateIdentityCertNameAndDistributor( identity_cert_watcher_); } if (identity_cert_distributor != nullptr) { - UpdateIdentityCertWatcher(identity_cert_distributor.get()); + UpdateIdentityCertWatcher(cert_name, identity_cert_distributor.get()); } else { identity_cert_watcher_ = nullptr; - distributor_->SetErrorForCert( + xds_certificate_provider_->distributor_->SetErrorForCert( "", absl::nullopt, GRPC_ERROR_CREATE_FROM_STATIC_STRING( "No certificate provider available for identity certificates")); @@ -176,42 +181,45 @@ void XdsCertificateProvider::UpdateIdentityCertNameAndDistributor( identity_cert_distributor_ = std::move(identity_cert_distributor); } -void XdsCertificateProvider::UpdateSubjectAlternativeNameMatchers( - std::vector<XdsApi::StringMatcher> matchers) { - MutexLock lock(&san_matchers_mu_); - san_matchers_ = std::move(matchers); +void XdsCertificateProvider::ClusterCertificateState::UpdateRootCertWatcher( + const std::string& cert_name, + grpc_tls_certificate_distributor* root_cert_distributor) { + auto watcher = absl::make_unique<RootCertificatesWatcher>( + xds_certificate_provider_->distributor_, cert_name); + root_cert_watcher_ = watcher.get(); + root_cert_distributor->WatchTlsCertificates(std::move(watcher), + root_cert_name_, absl::nullopt); } -void XdsCertificateProvider::WatchStatusCallback(std::string cert_name, - bool root_being_watched, - bool identity_being_watched) { +void XdsCertificateProvider::ClusterCertificateState::UpdateIdentityCertWatcher( + const std::string& cert_name, + grpc_tls_certificate_distributor* identity_cert_distributor) { + auto watcher = absl::make_unique<IdentityCertificatesWatcher>( + xds_certificate_provider_->distributor_, cert_name); + identity_cert_watcher_ = watcher.get(); + identity_cert_distributor->WatchTlsCertificates( + std::move(watcher), absl::nullopt, identity_cert_name_); +} + +void XdsCertificateProvider::ClusterCertificateState::WatchStatusCallback( + const std::string& cert_name, bool root_being_watched, + bool identity_being_watched) { // We aren't specially handling the case where root_cert_distributor is same // as identity_cert_distributor. Always using two separate watchers // irrespective of the fact results in a straightforward design, and using a // single watcher does not seem to provide any benefit other than cutting down // on the number of callbacks. - MutexLock lock(&mu_); - if (!cert_name.empty()) { - grpc_error* error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( - absl::StrCat("Illegal certificate name: \'", cert_name, - "\'. Should be empty.") - .c_str()); - distributor_->SetErrorForCert(cert_name, GRPC_ERROR_REF(error), - GRPC_ERROR_REF(error)); - GRPC_ERROR_UNREF(error); - return; - } if (root_being_watched && !watching_root_certs_) { // We need to start watching root certs. watching_root_certs_ = true; if (root_cert_distributor_ == nullptr) { - distributor_->SetErrorForCert( - "", + xds_certificate_provider_->distributor_->SetErrorForCert( + cert_name, GRPC_ERROR_CREATE_FROM_STATIC_STRING( "No certificate provider available for root certificates"), absl::nullopt); } else { - UpdateRootCertWatcher(root_cert_distributor_.get()); + UpdateRootCertWatcher(cert_name, root_cert_distributor_.get()); } } else if (!root_being_watched && watching_root_certs_) { // We need to cancel root certs watch. @@ -225,12 +233,12 @@ void XdsCertificateProvider::WatchStatusCallback(std::string cert_name, if (identity_being_watched && !watching_identity_certs_) { watching_identity_certs_ = true; if (identity_cert_distributor_ == nullptr) { - distributor_->SetErrorForCert( - "", absl::nullopt, + xds_certificate_provider_->distributor_->SetErrorForCert( + cert_name, absl::nullopt, GRPC_ERROR_CREATE_FROM_STATIC_STRING( "No certificate provider available for identity certificates")); } else { - UpdateIdentityCertWatcher(identity_cert_distributor_.get()); + UpdateIdentityCertWatcher(cert_name, identity_cert_distributor_.get()); } } else if (!identity_being_watched && watching_identity_certs_) { watching_identity_certs_ = false; @@ -243,20 +251,118 @@ void XdsCertificateProvider::WatchStatusCallback(std::string cert_name, } } -void XdsCertificateProvider::UpdateRootCertWatcher( - grpc_tls_certificate_distributor* root_cert_distributor) { - auto watcher = absl::make_unique<RootCertificatesWatcher>(distributor()); - root_cert_watcher_ = watcher.get(); - root_cert_distributor->WatchTlsCertificates(std::move(watcher), - root_cert_name_, absl::nullopt); +// +// XdsCertificateProvider +// + +XdsCertificateProvider::XdsCertificateProvider() + : distributor_(MakeRefCounted<grpc_tls_certificate_distributor>()) { + distributor_->SetWatchStatusCallback( + absl::bind_front(&XdsCertificateProvider::WatchStatusCallback, this)); } -void XdsCertificateProvider::UpdateIdentityCertWatcher( - grpc_tls_certificate_distributor* identity_cert_distributor) { - auto watcher = absl::make_unique<IdentityCertificatesWatcher>(distributor()); - identity_cert_watcher_ = watcher.get(); - identity_cert_distributor->WatchTlsCertificates( - std::move(watcher), absl::nullopt, identity_cert_name_); +XdsCertificateProvider::~XdsCertificateProvider() { + distributor_->SetWatchStatusCallback(nullptr); +} + +bool XdsCertificateProvider::ProvidesRootCerts(const std::string& cert_name) { + MutexLock lock(&mu_); + auto it = certificate_state_map_.find(cert_name); + if (it == certificate_state_map_.end()) return false; + return it->second->ProvidesRootCerts(); +} + +void XdsCertificateProvider::UpdateRootCertNameAndDistributor( + const std::string& cert_name, absl::string_view root_cert_name, + RefCountedPtr<grpc_tls_certificate_distributor> root_cert_distributor) { + MutexLock lock(&mu_); + auto it = certificate_state_map_.find(cert_name); + if (it == certificate_state_map_.end()) { + it = certificate_state_map_ + .emplace(cert_name, + absl::make_unique<ClusterCertificateState>(this)) + .first; + } + it->second->UpdateRootCertNameAndDistributor(cert_name, root_cert_name, + root_cert_distributor); + // Delete unused entries. + if (it->second->IsSafeToRemove()) certificate_state_map_.erase(it); +} + +bool XdsCertificateProvider::ProvidesIdentityCerts( + const std::string& cert_name) { + MutexLock lock(&mu_); + auto it = certificate_state_map_.find(cert_name); + if (it == certificate_state_map_.end()) return false; + return it->second->ProvidesIdentityCerts(); +} + +void XdsCertificateProvider::UpdateIdentityCertNameAndDistributor( + const std::string& cert_name, absl::string_view identity_cert_name, + RefCountedPtr<grpc_tls_certificate_distributor> identity_cert_distributor) { + MutexLock lock(&mu_); + auto it = certificate_state_map_.find(cert_name); + if (it == certificate_state_map_.end()) { + it = certificate_state_map_ + .emplace(cert_name, + absl::make_unique<ClusterCertificateState>(this)) + .first; + } + it->second->UpdateIdentityCertNameAndDistributor( + cert_name, identity_cert_name, identity_cert_distributor); + // Delete unused entries. + if (it->second->IsSafeToRemove()) certificate_state_map_.erase(it); +} + +bool XdsCertificateProvider::GetRequireClientCertificate( + const std::string& cert_name) { + MutexLock lock(&mu_); + auto it = certificate_state_map_.find(cert_name); + if (it == certificate_state_map_.end()) return false; + return it->second->require_client_certificate(); +} + +void XdsCertificateProvider::UpdateRequireClientCertificate( + const std::string& cert_name, bool require_client_certificate) { + MutexLock lock(&mu_); + auto it = certificate_state_map_.find(cert_name); + if (it == certificate_state_map_.end()) return; + it->second->set_require_client_certificate(require_client_certificate); +} + +std::vector<StringMatcher> XdsCertificateProvider::GetSanMatchers( + const std::string& cluster) { + MutexLock lock(&san_matchers_mu_); + auto it = san_matcher_map_.find(cluster); + if (it == san_matcher_map_.end()) return {}; + return it->second; +} + +void XdsCertificateProvider::UpdateSubjectAlternativeNameMatchers( + const std::string& cluster, std::vector<StringMatcher> matchers) { + MutexLock lock(&san_matchers_mu_); + if (matchers.empty()) { + san_matcher_map_.erase(cluster); + } else { + san_matcher_map_[cluster] = std::move(matchers); + } +} + +void XdsCertificateProvider::WatchStatusCallback(std::string cert_name, + bool root_being_watched, + bool identity_being_watched) { + MutexLock lock(&mu_); + auto it = certificate_state_map_.find(cert_name); + if (it == certificate_state_map_.end()) { + it = certificate_state_map_ + .emplace(cert_name, + absl::make_unique<ClusterCertificateState>(this)) + .first; + } + it->second->WatchStatusCallback(cert_name, root_being_watched, + identity_being_watched); + // Delete unused entries. + if (it->second->IsSafeToRemove()) certificate_state_map_.erase(it); } namespace { diff --git a/grpc/src/core/ext/xds/xds_certificate_provider.h b/grpc/src/core/ext/xds/xds_certificate_provider.h index 4d13423a..2f508830 100644 --- a/grpc/src/core/ext/xds/xds_certificate_provider.h +++ b/grpc/src/core/ext/xds/xds_certificate_provider.h @@ -31,44 +31,34 @@ namespace grpc_core { class XdsCertificateProvider : public grpc_tls_certificate_provider { public: - XdsCertificateProvider( - absl::string_view root_cert_name, - RefCountedPtr<grpc_tls_certificate_distributor> root_cert_distributor, - absl::string_view identity_cert_name, - RefCountedPtr<grpc_tls_certificate_distributor> identity_cert_distributor, - std::vector<XdsApi::StringMatcher> san_matchers); - + XdsCertificateProvider(); ~XdsCertificateProvider() override; - void UpdateRootCertNameAndDistributor( - absl::string_view root_cert_name, - RefCountedPtr<grpc_tls_certificate_distributor> root_cert_distributor); - void UpdateIdentityCertNameAndDistributor( - absl::string_view identity_cert_name, - RefCountedPtr<grpc_tls_certificate_distributor> - identity_cert_distributor); - void UpdateSubjectAlternativeNameMatchers( - std::vector<XdsApi::StringMatcher> matchers); - grpc_core::RefCountedPtr<grpc_tls_certificate_distributor> distributor() const override { return distributor_; } - bool ProvidesRootCerts() { - MutexLock lock(&mu_); - return root_cert_distributor_ != nullptr; - } + bool ProvidesRootCerts(const std::string& cert_name); + void UpdateRootCertNameAndDistributor( + const std::string& cert_name, absl::string_view root_cert_name, + RefCountedPtr<grpc_tls_certificate_distributor> root_cert_distributor); - bool ProvidesIdentityCerts() { - MutexLock lock(&mu_); - return identity_cert_distributor_ != nullptr; - } + bool ProvidesIdentityCerts(const std::string& cert_name); + void UpdateIdentityCertNameAndDistributor( + const std::string& cert_name, absl::string_view identity_cert_name, + RefCountedPtr<grpc_tls_certificate_distributor> + identity_cert_distributor); - std::vector<XdsApi::StringMatcher> subject_alternative_name_matchers() { - MutexLock lock(&san_matchers_mu_); - return san_matchers_; - } + bool GetRequireClientCertificate(const std::string& cert_name); + // Updating \a require_client_certificate for a non-existing \a cert_name has + // no effect. + void UpdateRequireClientCertificate(const std::string& cert_name, + bool require_client_certificate); + + std::vector<StringMatcher> GetSanMatchers(const std::string& cluster); + void UpdateSubjectAlternativeNameMatchers( + const std::string& cluster, std::vector<StringMatcher> matchers); grpc_arg MakeChannelArg() const; @@ -76,14 +66,73 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { const grpc_channel_args* args); private: + class ClusterCertificateState { + public: + explicit ClusterCertificateState( + XdsCertificateProvider* xds_certificate_provider) + : xds_certificate_provider_(xds_certificate_provider) {} + + ~ClusterCertificateState(); + + // Returns true if the certs aren't being watched and there are no + // distributors configured. + bool IsSafeToRemove() const; + + bool ProvidesRootCerts() const { return root_cert_distributor_ != nullptr; } + bool ProvidesIdentityCerts() const { + return identity_cert_distributor_ != nullptr; + } + + void UpdateRootCertNameAndDistributor( + const std::string& cert_name, absl::string_view root_cert_name, + RefCountedPtr<grpc_tls_certificate_distributor> root_cert_distributor); + void UpdateIdentityCertNameAndDistributor( + const std::string& cert_name, absl::string_view identity_cert_name, + RefCountedPtr<grpc_tls_certificate_distributor> + identity_cert_distributor); + + void UpdateRootCertWatcher( + const std::string& cert_name, + grpc_tls_certificate_distributor* root_cert_distributor); + void UpdateIdentityCertWatcher( + const std::string& cert_name, + grpc_tls_certificate_distributor* identity_cert_distributor); + + bool require_client_certificate() const { + return require_client_certificate_; + } + void set_require_client_certificate(bool require_client_certificate) { + require_client_certificate_ = require_client_certificate; + } + + void WatchStatusCallback(const std::string& cert_name, + bool root_being_watched, + bool identity_being_watched); + + private: + XdsCertificateProvider* xds_certificate_provider_; + bool watching_root_certs_ = false; + bool watching_identity_certs_ = false; + std::string root_cert_name_; + std::string identity_cert_name_; + RefCountedPtr<grpc_tls_certificate_distributor> root_cert_distributor_; + RefCountedPtr<grpc_tls_certificate_distributor> identity_cert_distributor_; + grpc_tls_certificate_distributor::TlsCertificatesWatcherInterface* + root_cert_watcher_ = nullptr; + grpc_tls_certificate_distributor::TlsCertificatesWatcherInterface* + identity_cert_watcher_ = nullptr; + bool require_client_certificate_ = false; + }; + void WatchStatusCallback(std::string cert_name, bool root_being_watched, bool identity_being_watched); - void UpdateRootCertWatcher( - grpc_tls_certificate_distributor* root_cert_distributor); - void UpdateIdentityCertWatcher( - grpc_tls_certificate_distributor* identity_cert_distributor); + + RefCountedPtr<grpc_tls_certificate_distributor> distributor_; Mutex mu_; + std::map<std::string /*cert_name*/, std::unique_ptr<ClusterCertificateState>> + certificate_state_map_ ABSL_GUARDED_BY(mu_); + // Use a separate mutex for san_matchers_ to avoid deadlocks since // san_matchers_ needs to be accessed when a handshake is being done and we // run into a possible deadlock scenario if using the same mutex. The mutex @@ -93,18 +142,8 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { // -> HandshakeManager::Add() -> SecurityHandshaker::DoHandshake() -> // subject_alternative_names_matchers() Mutex san_matchers_mu_; - bool watching_root_certs_ = false; - bool watching_identity_certs_ = false; - std::string root_cert_name_; - std::string identity_cert_name_; - RefCountedPtr<grpc_tls_certificate_distributor> root_cert_distributor_; - RefCountedPtr<grpc_tls_certificate_distributor> identity_cert_distributor_; - std::vector<XdsApi::StringMatcher> san_matchers_; - RefCountedPtr<grpc_tls_certificate_distributor> distributor_; - grpc_tls_certificate_distributor::TlsCertificatesWatcherInterface* - root_cert_watcher_ = nullptr; - grpc_tls_certificate_distributor::TlsCertificatesWatcherInterface* - identity_cert_watcher_ = nullptr; + std::map<std::string /*cluster_name*/, std::vector<StringMatcher>> + san_matcher_map_ ABSL_GUARDED_BY(san_matchers_mu_); }; } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_channel_args.h b/grpc/src/core/ext/xds/xds_channel_args.h index a2589403..ea6c862b 100644 --- a/grpc/src/core/ext/xds/xds_channel_args.h +++ b/grpc/src/core/ext/xds/xds_channel_args.h @@ -17,8 +17,11 @@ #ifndef GRPC_CORE_EXT_XDS_XDS_CHANNEL_ARGS_H #define GRPC_CORE_EXT_XDS_XDS_CHANNEL_ARGS_H -// Pointer channel arg containing a ref to the XdsClient object. -#define GRPC_ARG_XDS_CLIENT "grpc.xds_client" +// Specifies channel args for the xDS client. +// Used only when GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_BOOTSTRAP_CONFIG +// is set. +#define GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS \ + "grpc.xds_client_channel_args" // Timeout in milliseconds to wait for a resource to be returned from // the xds server before assuming that it does not exist. diff --git a/grpc/src/core/ext/xds/xds_client.cc b/grpc/src/core/ext/xds/xds_client.cc index b31bbbdc..d3549997 100644 --- a/grpc/src/core/ext/xds/xds_client.cc +++ b/grpc/src/core/ext/xds/xds_client.cc @@ -1,20 +1,18 @@ -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// #include <grpc/support/port_platform.h> @@ -35,19 +33,22 @@ #include "src/core/ext/filters/client_channel/client_channel.h" #include "src/core/ext/filters/client_channel/service_config.h" #include "src/core/ext/xds/xds_api.h" +#include "src/core/ext/xds/xds_bootstrap.h" #include "src/core/ext/xds/xds_channel_args.h" #include "src/core/ext/xds/xds_client.h" #include "src/core/ext/xds/xds_client_stats.h" +#include "src/core/ext/xds/xds_http_filters.h" +#include "src/core/lib/address_utils/sockaddr_utils.h" #include "src/core/lib/backoff/backoff.h" #include "src/core/lib/channel/channel_args.h" #include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/gpr/env.h" #include "src/core/lib/gpr/string.h" #include "src/core/lib/gprpp/memory.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/sync.h" #include "src/core/lib/iomgr/sockaddr.h" -#include "src/core/lib/iomgr/sockaddr_utils.h" #include "src/core/lib/iomgr/timer.h" #include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/slice/slice_string_helpers.h" @@ -70,8 +71,9 @@ TraceFlag grpc_xds_client_refcount_trace(false, "xds_client_refcount"); namespace { Mutex* g_mu = nullptr; -const grpc_channel_args* g_channel_args = nullptr; -XdsClient* g_xds_client = nullptr; +const grpc_channel_args* g_channel_args ABSL_GUARDED_BY(*g_mu) = nullptr; +XdsClient* g_xds_client ABSL_GUARDED_BY(*g_mu) = nullptr; +char* g_fallback_bootstrap_config ABSL_GUARDED_BY(*g_mu) = nullptr; } // namespace @@ -99,8 +101,8 @@ class XdsClient::ChannelState::RetryableCall private: void StartNewCallLocked(); void StartRetryTimerLocked(); - static void OnRetryTimer(void* arg, grpc_error* error); - void OnRetryTimerLocked(grpc_error* error); + static void OnRetryTimer(void* arg, grpc_error_handle error); + void OnRetryTimerLocked(grpc_error_handle error); // The wrapped xds call that talks to the xds server. It's instantiated // every time we start a new call. It's null during call retry backoff. @@ -132,9 +134,11 @@ class XdsClient::ChannelState::AdsCallState XdsClient* xds_client() const { return chand()->xds_client(); } bool seen_response() const { return seen_response_; } - void Subscribe(const std::string& type_url, const std::string& name); - void Unsubscribe(const std::string& type_url, const std::string& name, - bool delay_unsubscription); + void SubscribeLocked(const std::string& type_url, const std::string& name) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + void UnsubscribeLocked(const std::string& type_url, const std::string& name, + bool delay_unsubscription) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); bool HasSubscribedResources() const; @@ -175,7 +179,7 @@ class XdsClient::ChannelState::AdsCallState } private: - static void OnTimer(void* arg, grpc_error* error) { + static void OnTimer(void* arg, grpc_error_handle error) { ResourceState* self = static_cast<ResourceState*>(arg); { MutexLock lock(&self->ads_calld_->xds_client()->mu_); @@ -185,36 +189,43 @@ class XdsClient::ChannelState::AdsCallState self->Unref(DEBUG_LOCATION, "timer"); } - void OnTimerLocked(grpc_error* error) { + void OnTimerLocked(grpc_error_handle error) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) { if (error == GRPC_ERROR_NONE && timer_pending_) { timer_pending_ = false; - grpc_error* watcher_error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( + grpc_error_handle watcher_error = GRPC_ERROR_CREATE_FROM_COPIED_STRING( absl::StrFormat( "timeout obtaining resource {type=%s name=%s} from xds server", type_url_, name_) .c_str()); + watcher_error = grpc_error_set_int( + watcher_error, GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] %s", ads_calld_->xds_client(), - grpc_error_string(watcher_error)); + grpc_error_std_string(watcher_error).c_str()); } if (type_url_ == XdsApi::kLdsTypeUrl) { ListenerState& state = ads_calld_->xds_client()->listener_map_[name_]; + state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST; for (const auto& p : state.watchers) { p.first->OnError(GRPC_ERROR_REF(watcher_error)); } } else if (type_url_ == XdsApi::kRdsTypeUrl) { RouteConfigState& state = ads_calld_->xds_client()->route_config_map_[name_]; + state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST; for (const auto& p : state.watchers) { p.first->OnError(GRPC_ERROR_REF(watcher_error)); } } else if (type_url_ == XdsApi::kCdsTypeUrl) { ClusterState& state = ads_calld_->xds_client()->cluster_map_[name_]; + state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST; for (const auto& p : state.watchers) { p.first->OnError(GRPC_ERROR_REF(watcher_error)); } } else if (type_url_ == XdsApi::kEdsTypeUrl) { EndpointState& state = ads_calld_->xds_client()->endpoint_map_[name_]; + state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST; for (const auto& p : state.watchers) { p.first->OnError(GRPC_ERROR_REF(watcher_error)); } @@ -241,26 +252,38 @@ class XdsClient::ChannelState::AdsCallState // Nonce and error for this resource type. std::string nonce; - grpc_error* error = GRPC_ERROR_NONE; + grpc_error_handle error = GRPC_ERROR_NONE; // Subscribed resources of this type. std::map<std::string /* name */, OrphanablePtr<ResourceState>> subscribed_resources; }; - void SendMessageLocked(const std::string& type_url); - - void AcceptLdsUpdate(XdsApi::LdsUpdateMap lds_update_map); - void AcceptRdsUpdate(XdsApi::RdsUpdateMap rds_update_map); - void AcceptCdsUpdate(XdsApi::CdsUpdateMap cds_update_map); - void AcceptEdsUpdate(XdsApi::EdsUpdateMap eds_update_map); - - static void OnRequestSent(void* arg, grpc_error* error); - void OnRequestSentLocked(grpc_error* error); - static void OnResponseReceived(void* arg, grpc_error* error); - bool OnResponseReceivedLocked(); - static void OnStatusReceived(void* arg, grpc_error* error); - void OnStatusReceivedLocked(grpc_error* error); + void SendMessageLocked(const std::string& type_url) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + + void AcceptLdsUpdateLocked(std::string version, grpc_millis update_time, + XdsApi::LdsUpdateMap lds_update_map) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + void AcceptRdsUpdateLocked(std::string version, grpc_millis update_time, + XdsApi::RdsUpdateMap rds_update_map) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + void AcceptCdsUpdateLocked(std::string version, grpc_millis update_time, + XdsApi::CdsUpdateMap cds_update_map) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + void AcceptEdsUpdateLocked(std::string version, grpc_millis update_time, + XdsApi::EdsUpdateMap eds_update_map) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + + static void OnRequestSent(void* arg, grpc_error_handle error); + void OnRequestSentLocked(grpc_error_handle error) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + static void OnResponseReceived(void* arg, grpc_error_handle error); + bool OnResponseReceivedLocked() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + static void OnStatusReceived(void* arg, grpc_error_handle error); + void OnStatusReceivedLocked(grpc_error_handle error) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); bool IsCurrentCallOnChannel() const; @@ -333,12 +356,15 @@ class XdsClient::ChannelState::LrsCallState void Orphan() override; private: - void ScheduleNextReportLocked(); - static void OnNextReportTimer(void* arg, grpc_error* error); - bool OnNextReportTimerLocked(grpc_error* error); - bool SendReportLocked(); - static void OnReportDone(void* arg, grpc_error* error); - bool OnReportDoneLocked(grpc_error* error); + void ScheduleNextReportLocked() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + static void OnNextReportTimer(void* arg, grpc_error_handle error); + bool OnNextReportTimerLocked(grpc_error_handle error) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + bool SendReportLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + static void OnReportDone(void* arg, grpc_error_handle error); + bool OnReportDoneLocked(grpc_error_handle error) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); bool IsCurrentReporterOnCall() const { return this == parent_->reporter_.get(); @@ -357,12 +383,15 @@ class XdsClient::ChannelState::LrsCallState grpc_closure on_report_done_; }; - static void OnInitialRequestSent(void* arg, grpc_error* error); - void OnInitialRequestSentLocked(); - static void OnResponseReceived(void* arg, grpc_error* error); - bool OnResponseReceivedLocked(); - static void OnStatusReceived(void* arg, grpc_error* error); - void OnStatusReceivedLocked(grpc_error* error); + static void OnInitialRequestSent(void* arg, grpc_error_handle error); + void OnInitialRequestSentLocked() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + static void OnResponseReceived(void* arg, grpc_error_handle error); + bool OnResponseReceivedLocked() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + static void OnStatusReceived(void* arg, grpc_error_handle error); + void OnStatusReceivedLocked(grpc_error_handle error) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); bool IsCurrentCallOnChannel() const; @@ -418,7 +447,7 @@ class XdsClient::ChannelState::StateWatcher "[xds_client %p] xds channel in state:TRANSIENT_FAILURE " "status_message:(%s)", parent_->xds_client(), status.ToString().c_str()); - parent_->xds_client()->NotifyOnErrorLocked( + parent_->xds_client_->NotifyOnErrorLocked( GRPC_ERROR_CREATE_FROM_STATIC_STRING( "xds channel in TRANSIENT_FAILURE")); } @@ -433,26 +462,13 @@ class XdsClient::ChannelState::StateWatcher namespace { -grpc_channel* CreateXdsChannel(const XdsBootstrap::XdsServer& server) { - // Build channel args. - absl::InlinedVector<grpc_arg, 2> args_to_add = { - grpc_channel_arg_integer_create( - const_cast<char*>(GRPC_ARG_KEEPALIVE_TIME_MS), - 5 * 60 * GPR_MS_PER_SEC), - grpc_channel_arg_integer_create( - const_cast<char*>(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL), 1), - }; - grpc_channel_args* new_args = grpc_channel_args_copy_and_add( - g_channel_args, args_to_add.data(), args_to_add.size()); - // Create channel creds. +grpc_channel* CreateXdsChannel(grpc_channel_args* args, + const XdsBootstrap::XdsServer& server) { RefCountedPtr<grpc_channel_credentials> channel_creds = XdsChannelCredsRegistry::MakeChannelCreds(server.channel_creds_type, server.channel_creds_config); - // Create channel. - grpc_channel* channel = grpc_secure_channel_create( - channel_creds.get(), server.server_uri.c_str(), new_args, nullptr); - grpc_channel_args_destroy(new_args); - return channel; + return grpc_secure_channel_create(channel_creds.get(), + server.server_uri.c_str(), args, nullptr); } } // namespace @@ -469,7 +485,7 @@ XdsClient::ChannelState::ChannelState(WeakRefCountedPtr<XdsClient> xds_client, gpr_log(GPR_INFO, "[xds_client %p] creating channel to %s", xds_client_.get(), server.server_uri.c_str()); } - channel_ = CreateXdsChannel(server); + channel_ = CreateXdsChannel(xds_client_->args_, server); GPR_ASSERT(channel_ != nullptr); StartConnectivityWatchLocked(); } @@ -502,7 +518,7 @@ XdsClient::ChannelState::LrsCallState* XdsClient::ChannelState::lrs_calld() } bool XdsClient::ChannelState::HasActiveAdsCall() const { - return ads_calld_->calld() != nullptr; + return ads_calld_ != nullptr && ads_calld_->calld() != nullptr; } void XdsClient::ChannelState::MaybeStartLrsCall() { @@ -514,24 +530,22 @@ void XdsClient::ChannelState::MaybeStartLrsCall() { void XdsClient::ChannelState::StopLrsCall() { lrs_calld_.reset(); } void XdsClient::ChannelState::StartConnectivityWatchLocked() { - grpc_channel_element* client_channel_elem = - grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel_)); - GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter); + ClientChannel* client_channel = ClientChannel::GetFromChannel(channel_); + GPR_ASSERT(client_channel != nullptr); watcher_ = new StateWatcher(Ref(DEBUG_LOCATION, "ChannelState+watch")); - grpc_client_channel_start_connectivity_watch( - client_channel_elem, GRPC_CHANNEL_IDLE, + client_channel->AddConnectivityWatcher( + GRPC_CHANNEL_IDLE, OrphanablePtr<AsyncConnectivityStateWatcherInterface>(watcher_)); } void XdsClient::ChannelState::CancelConnectivityWatchLocked() { - grpc_channel_element* client_channel_elem = - grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel_)); - GPR_ASSERT(client_channel_elem->filter == &grpc_client_channel_filter); - grpc_client_channel_stop_connectivity_watch(client_channel_elem, watcher_); + ClientChannel* client_channel = ClientChannel::GetFromChannel(channel_); + GPR_ASSERT(client_channel != nullptr); + client_channel->RemoveConnectivityWatcher(watcher_); } -void XdsClient::ChannelState::Subscribe(const std::string& type_url, - const std::string& name) { +void XdsClient::ChannelState::SubscribeLocked(const std::string& type_url, + const std::string& name) { if (ads_calld_ == nullptr) { // Start the ADS call if this is the first request. ads_calld_.reset(new RetryableCall<AdsCallState>( @@ -545,16 +559,16 @@ void XdsClient::ChannelState::Subscribe(const std::string& type_url, // because when the call is restarted it will resend all necessary requests. if (ads_calld() == nullptr) return; // Subscribe to this resource if the ADS call is active. - ads_calld()->Subscribe(type_url, name); + ads_calld()->SubscribeLocked(type_url, name); } -void XdsClient::ChannelState::Unsubscribe(const std::string& type_url, - const std::string& name, - bool delay_unsubscription) { +void XdsClient::ChannelState::UnsubscribeLocked(const std::string& type_url, + const std::string& name, + bool delay_unsubscription) { if (ads_calld_ != nullptr) { auto* calld = ads_calld_->calld(); if (calld != nullptr) { - calld->Unsubscribe(type_url, name, delay_unsubscription); + calld->UnsubscribeLocked(type_url, name, delay_unsubscription); if (!calld->HasSubscribedResources()) ads_calld_.reset(); } } @@ -637,7 +651,7 @@ void XdsClient::ChannelState::RetryableCall<T>::StartRetryTimerLocked() { template <typename T> void XdsClient::ChannelState::RetryableCall<T>::OnRetryTimer( - void* arg, grpc_error* error) { + void* arg, grpc_error_handle error) { RetryableCall* calld = static_cast<RetryableCall*>(arg); { MutexLock lock(&calld->chand_->xds_client()->mu_); @@ -648,7 +662,7 @@ void XdsClient::ChannelState::RetryableCall<T>::OnRetryTimer( template <typename T> void XdsClient::ChannelState::RetryableCall<T>::OnRetryTimerLocked( - grpc_error* error) { + grpc_error_handle error) { retry_timer_callback_pending_ = false; if (!shutting_down_ && error == GRPC_ERROR_NONE) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { @@ -716,16 +730,16 @@ XdsClient::ChannelState::AdsCallState::AdsCallState( GRPC_CLOSURE_INIT(&on_request_sent_, OnRequestSent, this, grpc_schedule_on_exec_ctx); for (const auto& p : xds_client()->listener_map_) { - Subscribe(XdsApi::kLdsTypeUrl, std::string(p.first)); + SubscribeLocked(XdsApi::kLdsTypeUrl, std::string(p.first)); } for (const auto& p : xds_client()->route_config_map_) { - Subscribe(XdsApi::kRdsTypeUrl, std::string(p.first)); + SubscribeLocked(XdsApi::kRdsTypeUrl, std::string(p.first)); } for (const auto& p : xds_client()->cluster_map_) { - Subscribe(XdsApi::kCdsTypeUrl, std::string(p.first)); + SubscribeLocked(XdsApi::kCdsTypeUrl, std::string(p.first)); } for (const auto& p : xds_client()->endpoint_map_) { - Subscribe(XdsApi::kEdsTypeUrl, std::string(p.first)); + SubscribeLocked(XdsApi::kEdsTypeUrl, std::string(p.first)); } // Op: recv initial metadata. op = ops; @@ -789,7 +803,8 @@ void XdsClient::ChannelState::AdsCallState::Orphan() { } void XdsClient::ChannelState::AdsCallState::SendMessageLocked( - const std::string& type_url) { + const std::string& type_url) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) { // Buffer message sending if an existing message is in flight. if (send_message_payload_ != nullptr) { buffered_requests_.insert(type_url); @@ -814,7 +829,7 @@ void XdsClient::ChannelState::AdsCallState::SendMessageLocked( "error=%s resources=%s", xds_client(), type_url.c_str(), xds_client()->resource_version_map_[type_url].c_str(), - state.nonce.c_str(), grpc_error_string(state.error), + state.nonce.c_str(), grpc_error_std_string(state.error).c_str(), absl::StrJoin(resource_names, " ").c_str()); } GRPC_ERROR_UNREF(state.error); @@ -841,7 +856,7 @@ void XdsClient::ChannelState::AdsCallState::SendMessageLocked( } } -void XdsClient::ChannelState::AdsCallState::Subscribe( +void XdsClient::ChannelState::AdsCallState::SubscribeLocked( const std::string& type_url, const std::string& name) { auto& state = state_map_[type_url].subscribed_resources[name]; if (state == nullptr) { @@ -851,7 +866,7 @@ void XdsClient::ChannelState::AdsCallState::Subscribe( } } -void XdsClient::ChannelState::AdsCallState::Unsubscribe( +void XdsClient::ChannelState::AdsCallState::UnsubscribeLocked( const std::string& type_url, const std::string& name, bool delay_unsubscription) { state_map_[type_url].subscribed_resources.erase(name); @@ -865,7 +880,24 @@ bool XdsClient::ChannelState::AdsCallState::HasSubscribedResources() const { return false; } -void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate( +namespace { + +// Build a resource metadata struct for ADS result accepting methods and CSDS. +XdsApi::ResourceMetadata CreateResourceMetadataAcked( + std::string serialized_proto, std::string version, + grpc_millis update_time) { + XdsApi::ResourceMetadata resource_metadata; + resource_metadata.serialized_proto = std::move(serialized_proto); + resource_metadata.update_time = update_time; + resource_metadata.version = std::move(version); + resource_metadata.client_status = XdsApi::ResourceMetadata::ACKED; + return resource_metadata; +} + +} // namespace + +void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdateLocked( + std::string version, grpc_millis update_time, XdsApi::LdsUpdateMap lds_update_map) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, @@ -877,23 +909,17 @@ void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate( std::set<std::string> rds_resource_names_seen; for (auto& p : lds_update_map) { const std::string& listener_name = p.first; - XdsApi::LdsUpdate& lds_update = p.second; + XdsApi::LdsUpdate& lds_update = p.second.resource; auto& state = lds_state.subscribed_resources[listener_name]; if (state != nullptr) state->Finish(); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log(GPR_INFO, "[xds_client %p] LDS resource %s: route_config_name=%s", - xds_client(), listener_name.c_str(), - (!lds_update.route_config_name.empty() - ? lds_update.route_config_name.c_str() - : "<inlined>")); - if (lds_update.rds_update.has_value()) { - gpr_log(GPR_INFO, "RouteConfiguration: %s", - lds_update.rds_update->ToString().c_str()); - } + gpr_log(GPR_INFO, "[xds_client %p] LDS resource %s: %s", xds_client(), + listener_name.c_str(), lds_update.ToString().c_str()); } // Record the RDS resource names seen. - if (!lds_update.route_config_name.empty()) { - rds_resource_names_seen.insert(lds_update.route_config_name); + if (!lds_update.http_connection_manager.route_config_name.empty()) { + rds_resource_names_seen.insert( + lds_update.http_connection_manager.route_config_name); } // Ignore identical update. ListenerState& listener_state = xds_client()->listener_map_[listener_name]; @@ -909,6 +935,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate( } // Update the listener state. listener_state.update = std::move(lds_update); + listener_state.meta = CreateResourceMetadataAcked( + std::move(p.second.serialized_proto), version, update_time); // Notify watchers. for (const auto& p : listener_state.watchers) { p.first->OnListenerChanged(*listener_state.update); @@ -952,7 +980,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptLdsUpdate( } } -void XdsClient::ChannelState::AdsCallState::AcceptRdsUpdate( +void XdsClient::ChannelState::AdsCallState::AcceptRdsUpdateLocked( + std::string version, grpc_millis update_time, XdsApi::RdsUpdateMap rds_update_map) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, @@ -963,7 +992,7 @@ void XdsClient::ChannelState::AdsCallState::AcceptRdsUpdate( auto& rds_state = state_map_[XdsApi::kRdsTypeUrl]; for (auto& p : rds_update_map) { const std::string& route_config_name = p.first; - XdsApi::RdsUpdate& rds_update = p.second; + XdsApi::RdsUpdate& rds_update = p.second.resource; auto& state = rds_state.subscribed_resources[route_config_name]; if (state != nullptr) state->Finish(); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { @@ -984,6 +1013,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptRdsUpdate( } // Update the cache. route_config_state.update = std::move(rds_update); + route_config_state.meta = CreateResourceMetadataAcked( + std::move(p.second.serialized_proto), version, update_time); // Notify all watchers. for (const auto& p : route_config_state.watchers) { p.first->OnRouteConfigChanged(*route_config_state.update); @@ -991,7 +1022,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptRdsUpdate( } } -void XdsClient::ChannelState::AdsCallState::AcceptCdsUpdate( +void XdsClient::ChannelState::AdsCallState::AcceptCdsUpdateLocked( + std::string version, grpc_millis update_time, XdsApi::CdsUpdateMap cds_update_map) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, @@ -1003,7 +1035,7 @@ void XdsClient::ChannelState::AdsCallState::AcceptCdsUpdate( std::set<std::string> eds_resource_names_seen; for (auto& p : cds_update_map) { const char* cluster_name = p.first.c_str(); - XdsApi::CdsUpdate& cds_update = p.second; + XdsApi::CdsUpdate& cds_update = p.second.resource; auto& state = cds_state.subscribed_resources[cluster_name]; if (state != nullptr) state->Finish(); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { @@ -1027,6 +1059,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptCdsUpdate( } // Update the cluster state. cluster_state.update = std::move(cds_update); + cluster_state.meta = CreateResourceMetadataAcked( + std::move(p.second.serialized_proto), version, update_time); // Notify all watchers. for (const auto& p : cluster_state.watchers) { p.first->OnClusterChanged(cluster_state.update.value()); @@ -1069,7 +1103,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptCdsUpdate( } } -void XdsClient::ChannelState::AdsCallState::AcceptEdsUpdate( +void XdsClient::ChannelState::AdsCallState::AcceptEdsUpdateLocked( + std::string version, grpc_millis update_time, XdsApi::EdsUpdateMap eds_update_map) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, @@ -1080,7 +1115,7 @@ void XdsClient::ChannelState::AdsCallState::AcceptEdsUpdate( auto& eds_state = state_map_[XdsApi::kEdsTypeUrl]; for (auto& p : eds_update_map) { const char* eds_service_name = p.first.c_str(); - XdsApi::EdsUpdate& eds_update = p.second; + XdsApi::EdsUpdate& eds_update = p.second.resource; auto& state = eds_state.subscribed_resources[eds_service_name]; if (state != nullptr) state->Finish(); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { @@ -1101,6 +1136,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptEdsUpdate( } // Update the cluster state. endpoint_state.update = std::move(eds_update); + endpoint_state.meta = CreateResourceMetadataAcked( + std::move(p.second.serialized_proto), version, update_time); // Notify all watchers. for (const auto& p : endpoint_state.watchers) { p.first->OnEndpointChanged(endpoint_state.update.value()); @@ -1108,8 +1145,8 @@ void XdsClient::ChannelState::AdsCallState::AcceptEdsUpdate( } } -void XdsClient::ChannelState::AdsCallState::OnRequestSent(void* arg, - grpc_error* error) { +void XdsClient::ChannelState::AdsCallState::OnRequestSent( + void* arg, grpc_error_handle error) { AdsCallState* ads_calld = static_cast<AdsCallState*>(arg); { MutexLock lock(&ads_calld->xds_client()->mu_); @@ -1119,7 +1156,7 @@ void XdsClient::ChannelState::AdsCallState::OnRequestSent(void* arg, } void XdsClient::ChannelState::AdsCallState::OnRequestSentLocked( - grpc_error* error) { + grpc_error_handle error) { if (IsCurrentCallOnChannel() && error == GRPC_ERROR_NONE) { // Clean up the sent message. grpc_byte_buffer_destroy(send_message_payload_); @@ -1143,7 +1180,7 @@ void XdsClient::ChannelState::AdsCallState::OnRequestSentLocked( } void XdsClient::ChannelState::AdsCallState::OnResponseReceived( - void* arg, grpc_error* /* error */) { + void* arg, grpc_error_handle /* error */) { AdsCallState* ads_calld = static_cast<AdsCallState*>(arg); bool done; { @@ -1167,7 +1204,8 @@ bool XdsClient::ChannelState::AdsCallState::OnResponseReceivedLocked() { recv_message_payload_ = nullptr; // Parse and validate the response. XdsApi::AdsParseResult result = xds_client()->api_.ParseAdsResponse( - response_slice, ResourceNamesForRequest(XdsApi::kLdsTypeUrl), + chand()->server_, response_slice, + ResourceNamesForRequest(XdsApi::kLdsTypeUrl), ResourceNamesForRequest(XdsApi::kRdsTypeUrl), ResourceNamesForRequest(XdsApi::kCdsTypeUrl), ResourceNamesForRequest(XdsApi::kEdsTypeUrl)); @@ -1176,14 +1214,17 @@ bool XdsClient::ChannelState::AdsCallState::OnResponseReceivedLocked() { // Ignore unparsable response. gpr_log(GPR_ERROR, "[xds_client %p] Error parsing ADS response (%s) -- ignoring", - xds_client(), grpc_error_string(result.parse_error)); + xds_client(), grpc_error_std_string(result.parse_error).c_str()); GRPC_ERROR_UNREF(result.parse_error); } else { + grpc_millis update_time = grpc_core::ExecCtx::Get()->Now(); // Update nonce. auto& state = state_map_[result.type_url]; state.nonce = std::move(result.nonce); // NACK or ACK the response. if (result.parse_error != GRPC_ERROR_NONE) { + xds_client()->UpdateResourceMetadataWithFailedParseResultLocked( + update_time, result); GRPC_ERROR_UNREF(state.error); state.error = result.parse_error; // NACK unacceptable update. @@ -1191,19 +1232,24 @@ bool XdsClient::ChannelState::AdsCallState::OnResponseReceivedLocked() { "[xds_client %p] ADS response invalid for resource type %s " "version %s, will NACK: nonce=%s error=%s", xds_client(), result.type_url.c_str(), result.version.c_str(), - state.nonce.c_str(), grpc_error_string(result.parse_error)); + state.nonce.c_str(), + grpc_error_std_string(result.parse_error).c_str()); SendMessageLocked(result.type_url); } else { seen_response_ = true; // Accept the ADS response according to the type_url. if (result.type_url == XdsApi::kLdsTypeUrl) { - AcceptLdsUpdate(std::move(result.lds_update_map)); + AcceptLdsUpdateLocked(result.version, update_time, + std::move(result.lds_update_map)); } else if (result.type_url == XdsApi::kRdsTypeUrl) { - AcceptRdsUpdate(std::move(result.rds_update_map)); + AcceptRdsUpdateLocked(result.version, update_time, + std::move(result.rds_update_map)); } else if (result.type_url == XdsApi::kCdsTypeUrl) { - AcceptCdsUpdate(std::move(result.cds_update_map)); + AcceptCdsUpdateLocked(result.version, update_time, + std::move(result.cds_update_map)); } else if (result.type_url == XdsApi::kEdsTypeUrl) { - AcceptEdsUpdate(std::move(result.eds_update_map)); + AcceptEdsUpdateLocked(result.version, update_time, + std::move(result.eds_update_map)); } xds_client()->resource_version_map_[result.type_url] = std::move(result.version); @@ -1234,7 +1280,7 @@ bool XdsClient::ChannelState::AdsCallState::OnResponseReceivedLocked() { } void XdsClient::ChannelState::AdsCallState::OnStatusReceived( - void* arg, grpc_error* error) { + void* arg, grpc_error_handle error) { AdsCallState* ads_calld = static_cast<AdsCallState*>(arg); { MutexLock lock(&ads_calld->xds_client()->mu_); @@ -1244,14 +1290,14 @@ void XdsClient::ChannelState::AdsCallState::OnStatusReceived( } void XdsClient::ChannelState::AdsCallState::OnStatusReceivedLocked( - grpc_error* error) { + grpc_error_handle error) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { char* status_details = grpc_slice_to_c_string(status_details_); gpr_log(GPR_INFO, "[xds_client %p] ADS call status received. Status = %d, details " "= '%s', (chand: %p, ads_calld: %p, call: %p), error '%s'", xds_client(), status_code_, status_details, chand(), this, call_, - grpc_error_string(error)); + grpc_error_std_string(error).c_str()); gpr_free(status_details); } // Ignore status from a stale call. @@ -1306,7 +1352,7 @@ void XdsClient::ChannelState::LrsCallState::Reporter:: } void XdsClient::ChannelState::LrsCallState::Reporter::OnNextReportTimer( - void* arg, grpc_error* error) { + void* arg, grpc_error_handle error) { Reporter* self = static_cast<Reporter*>(arg); bool done; { @@ -1317,7 +1363,7 @@ void XdsClient::ChannelState::LrsCallState::Reporter::OnNextReportTimer( } bool XdsClient::ChannelState::LrsCallState::Reporter::OnNextReportTimerLocked( - grpc_error* error) { + grpc_error_handle error) { next_report_timer_callback_pending_ = false; if (error != GRPC_ERROR_NONE || !IsCurrentReporterOnCall()) { GRPC_ERROR_UNREF(error); @@ -1382,7 +1428,7 @@ bool XdsClient::ChannelState::LrsCallState::Reporter::SendReportLocked() { } void XdsClient::ChannelState::LrsCallState::Reporter::OnReportDone( - void* arg, grpc_error* error) { + void* arg, grpc_error_handle error) { Reporter* self = static_cast<Reporter*>(arg); bool done; { @@ -1393,7 +1439,7 @@ void XdsClient::ChannelState::LrsCallState::Reporter::OnReportDone( } bool XdsClient::ChannelState::LrsCallState::Reporter::OnReportDoneLocked( - grpc_error* error) { + grpc_error_handle error) { grpc_byte_buffer_destroy(parent_->send_message_payload_); parent_->send_message_payload_ = nullptr; // If there are no more registered stats to report, cancel the call. @@ -1563,7 +1609,7 @@ void XdsClient::ChannelState::LrsCallState::MaybeStartReportingLocked() { } void XdsClient::ChannelState::LrsCallState::OnInitialRequestSent( - void* arg, grpc_error* /*error*/) { + void* arg, grpc_error_handle /*error*/) { LrsCallState* lrs_calld = static_cast<LrsCallState*>(arg); { MutexLock lock(&lrs_calld->xds_client()->mu_); @@ -1580,7 +1626,7 @@ void XdsClient::ChannelState::LrsCallState::OnInitialRequestSentLocked() { } void XdsClient::ChannelState::LrsCallState::OnResponseReceived( - void* arg, grpc_error* /*error*/) { + void* arg, grpc_error_handle /*error*/) { LrsCallState* lrs_calld = static_cast<LrsCallState*>(arg); bool done; { @@ -1608,13 +1654,13 @@ bool XdsClient::ChannelState::LrsCallState::OnResponseReceivedLocked() { bool send_all_clusters = false; std::set<std::string> new_cluster_names; grpc_millis new_load_reporting_interval; - grpc_error* parse_error = xds_client()->api_.ParseLrsResponse( + grpc_error_handle parse_error = xds_client()->api_.ParseLrsResponse( response_slice, &send_all_clusters, &new_cluster_names, &new_load_reporting_interval); if (parse_error != GRPC_ERROR_NONE) { gpr_log(GPR_ERROR, "[xds_client %p] LRS response parsing failed. error=%s", - xds_client(), grpc_error_string(parse_error)); + xds_client(), grpc_error_std_string(parse_error).c_str()); GRPC_ERROR_UNREF(parse_error); return; } @@ -1683,7 +1729,7 @@ bool XdsClient::ChannelState::LrsCallState::OnResponseReceivedLocked() { } void XdsClient::ChannelState::LrsCallState::OnStatusReceived( - void* arg, grpc_error* error) { + void* arg, grpc_error_handle error) { LrsCallState* lrs_calld = static_cast<LrsCallState*>(arg); { MutexLock lock(&lrs_calld->xds_client()->mu_); @@ -1693,7 +1739,7 @@ void XdsClient::ChannelState::LrsCallState::OnStatusReceived( } void XdsClient::ChannelState::LrsCallState::OnStatusReceivedLocked( - grpc_error* error) { + grpc_error_handle error) { GPR_ASSERT(call_ != nullptr); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { char* status_details = grpc_slice_to_c_string(status_details_); @@ -1701,7 +1747,7 @@ void XdsClient::ChannelState::LrsCallState::OnStatusReceivedLocked( "[xds_client %p] LRS call status received. Status = %d, details " "= '%s', (chand: %p, calld: %p, call: %p), error '%s'", xds_client(), status_code_, status_details, chand(), this, call_, - grpc_error_string(error)); + grpc_error_std_string(error).c_str()); gpr_free(status_details); } // Ignore status from a stale call. @@ -1726,36 +1772,41 @@ bool XdsClient::ChannelState::LrsCallState::IsCurrentCallOnChannel() const { namespace { -grpc_millis GetRequestTimeout() { +grpc_millis GetRequestTimeout(const grpc_channel_args* args) { return grpc_channel_args_find_integer( - g_channel_args, GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS, + args, GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS, {15000, 0, INT_MAX}); } +grpc_channel_args* ModifyChannelArgs(const grpc_channel_args* args) { + absl::InlinedVector<grpc_arg, 2> args_to_add = { + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_KEEPALIVE_TIME_MS), + 5 * 60 * GPR_MS_PER_SEC), + grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_CHANNELZ_IS_INTERNAL_CHANNEL), 1), + }; + return grpc_channel_args_copy_and_add(args, args_to_add.data(), + args_to_add.size()); +} + } // namespace -XdsClient::XdsClient(grpc_error** error) +XdsClient::XdsClient(std::unique_ptr<XdsBootstrap> bootstrap, + const grpc_channel_args* args) : DualRefCounted<XdsClient>( GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_refcount_trace) ? "XdsClient" : nullptr), - request_timeout_(GetRequestTimeout()), + bootstrap_(std::move(bootstrap)), + args_(ModifyChannelArgs(args)), + request_timeout_(GetRequestTimeout(args)), interested_parties_(grpc_pollset_set_create()), - bootstrap_( - XdsBootstrap::ReadFromFile(this, &grpc_xds_client_trace, error)), certificate_provider_store_(MakeOrphanable<CertificateProviderStore>( - bootstrap_ == nullptr - ? CertificateProviderStore::PluginDefinitionMap() - : bootstrap_->certificate_providers())), - api_(this, &grpc_xds_client_trace, - bootstrap_ == nullptr ? nullptr : bootstrap_->node()) { + bootstrap_->certificate_providers())), + api_(this, &grpc_xds_client_trace, bootstrap_->node()) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] creating xds client", this); } - if (*error != GRPC_ERROR_NONE) { - gpr_log(GPR_ERROR, "[xds_client %p] failed to read bootstrap file: %s", - this, grpc_error_string(*error)); - return; - } // Create ChannelState object. chand_ = MakeOrphanable<ChannelState>( WeakRef(DEBUG_LOCATION, "XdsClient+ChannelState"), bootstrap_->server()); @@ -1765,11 +1816,13 @@ XdsClient::~XdsClient() { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] destroying xds client", this); } + grpc_channel_args_destroy(args_); grpc_pollset_set_destroy(interested_parties_); } void XdsClient::AddChannelzLinkage( channelz::ChannelNode* parent_channelz_node) { + MutexLock lock(&mu_); channelz::ChannelNode* xds_channelz_node = grpc_channel_get_channelz_node(chand_->channel()); if (xds_channelz_node != nullptr) { @@ -1779,6 +1832,7 @@ void XdsClient::AddChannelzLinkage( void XdsClient::RemoveChannelzLinkage( channelz::ChannelNode* parent_channelz_node) { + MutexLock lock(&mu_); channelz::ChannelNode* xds_channelz_node = grpc_channel_get_channelz_node(chand_->channel()); if (xds_channelz_node != nullptr) { @@ -1829,7 +1883,7 @@ void XdsClient::WatchListenerData( } w->OnListenerChanged(*listener_state.update); } - chand_->Subscribe(XdsApi::kLdsTypeUrl, listener_name_str); + chand_->SubscribeLocked(XdsApi::kLdsTypeUrl, listener_name_str); } void XdsClient::CancelListenerDataWatch(absl::string_view listener_name, @@ -1844,8 +1898,8 @@ void XdsClient::CancelListenerDataWatch(absl::string_view listener_name, listener_state.watchers.erase(it); if (listener_state.watchers.empty()) { listener_map_.erase(listener_name_str); - chand_->Unsubscribe(XdsApi::kLdsTypeUrl, listener_name_str, - delay_unsubscription); + chand_->UnsubscribeLocked(XdsApi::kLdsTypeUrl, listener_name_str, + delay_unsubscription); } } } @@ -1869,7 +1923,7 @@ void XdsClient::WatchRouteConfigData( } w->OnRouteConfigChanged(*route_config_state.update); } - chand_->Subscribe(XdsApi::kRdsTypeUrl, route_config_name_str); + chand_->SubscribeLocked(XdsApi::kRdsTypeUrl, route_config_name_str); } void XdsClient::CancelRouteConfigDataWatch(absl::string_view route_config_name, @@ -1885,8 +1939,8 @@ void XdsClient::CancelRouteConfigDataWatch(absl::string_view route_config_name, route_config_state.watchers.erase(it); if (route_config_state.watchers.empty()) { route_config_map_.erase(route_config_name_str); - chand_->Unsubscribe(XdsApi::kRdsTypeUrl, route_config_name_str, - delay_unsubscription); + chand_->UnsubscribeLocked(XdsApi::kRdsTypeUrl, route_config_name_str, + delay_unsubscription); } } } @@ -1908,7 +1962,7 @@ void XdsClient::WatchClusterData( } w->OnClusterChanged(cluster_state.update.value()); } - chand_->Subscribe(XdsApi::kCdsTypeUrl, cluster_name_str); + chand_->SubscribeLocked(XdsApi::kCdsTypeUrl, cluster_name_str); } void XdsClient::CancelClusterDataWatch(absl::string_view cluster_name, @@ -1923,8 +1977,8 @@ void XdsClient::CancelClusterDataWatch(absl::string_view cluster_name, cluster_state.watchers.erase(it); if (cluster_state.watchers.empty()) { cluster_map_.erase(cluster_name_str); - chand_->Unsubscribe(XdsApi::kCdsTypeUrl, cluster_name_str, - delay_unsubscription); + chand_->UnsubscribeLocked(XdsApi::kCdsTypeUrl, cluster_name_str, + delay_unsubscription); } } } @@ -1946,7 +2000,7 @@ void XdsClient::WatchEndpointData( } w->OnEndpointChanged(endpoint_state.update.value()); } - chand_->Subscribe(XdsApi::kEdsTypeUrl, eds_service_name_str); + chand_->SubscribeLocked(XdsApi::kEdsTypeUrl, eds_service_name_str); } void XdsClient::CancelEndpointDataWatch(absl::string_view eds_service_name, @@ -1961,8 +2015,8 @@ void XdsClient::CancelEndpointDataWatch(absl::string_view eds_service_name, endpoint_state.watchers.erase(it); if (endpoint_state.watchers.empty()) { endpoint_map_.erase(eds_service_name_str); - chand_->Unsubscribe(XdsApi::kEdsTypeUrl, eds_service_name_str, - delay_unsubscription); + chand_->UnsubscribeLocked(XdsApi::kEdsTypeUrl, eds_service_name_str, + delay_unsubscription); } } } @@ -2089,7 +2143,7 @@ void XdsClient::ResetBackoff() { } } -void XdsClient::NotifyOnErrorLocked(grpc_error* error) { +void XdsClient::NotifyOnErrorLocked(grpc_error_handle error) { for (const auto& p : listener_map_) { const ListenerState& listener_state = p.second; for (const auto& p : listener_state.watchers) { @@ -2198,25 +2252,190 @@ XdsApi::ClusterLoadReportMap XdsClient::BuildLoadReportSnapshotLocked( return snapshot_map; } +void XdsClient::UpdateResourceMetadataWithFailedParseResultLocked( + grpc_millis update_time, const XdsApi::AdsParseResult& result) { + // ADS update is rejected and the resource names in the failed update is + // available. + std::string details = grpc_error_std_string(result.parse_error); + for (auto& name : result.resource_names_failed) { + XdsApi::ResourceMetadata* resource_metadata = nullptr; + if (result.type_url == XdsApi::kLdsTypeUrl) { + auto it = listener_map_.find(name); + if (it != listener_map_.end()) { + resource_metadata = &it->second.meta; + } + } else if (result.type_url == XdsApi::kRdsTypeUrl) { + auto it = route_config_map_.find(name); + if (route_config_map_.find(name) != route_config_map_.end()) { + resource_metadata = &it->second.meta; + } + } else if (result.type_url == XdsApi::kCdsTypeUrl) { + auto it = cluster_map_.find(name); + if (cluster_map_.find(name) != cluster_map_.end()) { + resource_metadata = &it->second.meta; + } + } else if (result.type_url == XdsApi::kEdsTypeUrl) { + auto it = endpoint_map_.find(name); + if (endpoint_map_.find(name) != endpoint_map_.end()) { + resource_metadata = &it->second.meta; + } + } + if (resource_metadata == nullptr) { + return; + } + resource_metadata->client_status = XdsApi::ResourceMetadata::NACKED; + resource_metadata->failed_version = result.version; + resource_metadata->failed_details = details; + resource_metadata->failed_update_time = update_time; + } +} + +std::string XdsClient::DumpClientConfigBinary() { + MutexLock lock(&mu_); + XdsApi::ResourceTypeMetadataMap resource_type_metadata_map; + // Update per-xds-type version if available, this version corresponding to the + // last successful ADS update version. + for (auto& p : resource_version_map_) { + resource_type_metadata_map[p.first].version = p.second; + } + // Collect resource metadata from listeners + auto& lds_map = + resource_type_metadata_map[XdsApi::kLdsTypeUrl].resource_metadata_map; + for (auto& p : listener_map_) { + lds_map[p.first] = &p.second.meta; + } + // Collect resource metadata from route configs + auto& rds_map = + resource_type_metadata_map[XdsApi::kRdsTypeUrl].resource_metadata_map; + for (auto& p : route_config_map_) { + rds_map[p.first] = &p.second.meta; + } + // Collect resource metadata from clusters + auto& cds_map = + resource_type_metadata_map[XdsApi::kCdsTypeUrl].resource_metadata_map; + for (auto& p : cluster_map_) { + cds_map[p.first] = &p.second.meta; + } + // Collect resource metadata from endpoints + auto& eds_map = + resource_type_metadata_map[XdsApi::kEdsTypeUrl].resource_metadata_map; + for (auto& p : endpoint_map_) { + eds_map[p.first] = &p.second.meta; + } + // Assemble config dump messages + return api_.AssembleClientConfig(resource_type_metadata_map); +} + // // accessors for global state // -void XdsClientGlobalInit() { g_mu = new Mutex; } +void XdsClientGlobalInit() { + g_mu = new Mutex; + XdsHttpFilterRegistry::Init(); +} -void XdsClientGlobalShutdown() { +// TODO(roth): Find a better way to clear the fallback config that does +// not require using ABSL_NO_THREAD_SAFETY_ANALYSIS. +void XdsClientGlobalShutdown() ABSL_NO_THREAD_SAFETY_ANALYSIS { + gpr_free(g_fallback_bootstrap_config); + g_fallback_bootstrap_config = nullptr; delete g_mu; g_mu = nullptr; + XdsHttpFilterRegistry::Shutdown(); } -RefCountedPtr<XdsClient> XdsClient::GetOrCreate(grpc_error** error) { - MutexLock lock(g_mu); - if (g_xds_client != nullptr) { - auto xds_client = g_xds_client->RefIfNonZero(); - if (xds_client != nullptr) return xds_client; +namespace { + +std::string GetBootstrapContents(const char* fallback_config, + grpc_error_handle* error) { + // First, try GRPC_XDS_BOOTSTRAP env var. + grpc_core::UniquePtr<char> path(gpr_getenv("GRPC_XDS_BOOTSTRAP")); + if (path != nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, + "Got bootstrap file location from GRPC_XDS_BOOTSTRAP " + "environment variable: %s", + path.get()); + } + grpc_slice contents; + *error = + grpc_load_file(path.get(), /*add_null_terminator=*/true, &contents); + if (*error != GRPC_ERROR_NONE) return ""; + std::string contents_str(StringViewFromSlice(contents)); + grpc_slice_unref_internal(contents); + return contents_str; + } + // Next, try GRPC_XDS_BOOTSTRAP_CONFIG env var. + grpc_core::UniquePtr<char> env_config( + gpr_getenv("GRPC_XDS_BOOTSTRAP_CONFIG")); + if (env_config != nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, + "Got bootstrap contents from GRPC_XDS_BOOTSTRAP_CONFIG " + "environment variable"); + } + return env_config.get(); + } + // Finally, try fallback config. + if (fallback_config != nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, "Got bootstrap contents from fallback config"); + } + return fallback_config; + } + // No bootstrap config found. + *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( + "Environment variables GRPC_XDS_BOOTSTRAP or GRPC_XDS_BOOTSTRAP_CONFIG " + "not defined"); + return ""; +} + +} // namespace + +RefCountedPtr<XdsClient> XdsClient::GetOrCreate(const grpc_channel_args* args, + grpc_error_handle* error) { + RefCountedPtr<XdsClient> xds_client; + // If getting bootstrap from channel args, create a local XdsClient + // instance for the channel or server instead of using the global instance. + const char* bootstrap_config = grpc_channel_args_find_string( + args, GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_BOOTSTRAP_CONFIG); + if (bootstrap_config != nullptr) { + std::unique_ptr<XdsBootstrap> bootstrap = + XdsBootstrap::Create(bootstrap_config, error); + if (*error == GRPC_ERROR_NONE) { + grpc_channel_args* xds_channel_args = + grpc_channel_args_find_pointer<grpc_channel_args>( + args, + GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS); + return MakeRefCounted<XdsClient>(std::move(bootstrap), xds_channel_args); + } + return nullptr; + } + // Otherwise, use the global instance. + { + MutexLock lock(g_mu); + if (g_xds_client != nullptr) { + auto xds_client = g_xds_client->RefIfNonZero(); + if (xds_client != nullptr) return xds_client; + } + // Find bootstrap contents. + std::string bootstrap_contents = + GetBootstrapContents(g_fallback_bootstrap_config, error); + if (*error != GRPC_ERROR_NONE) return nullptr; + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, "xDS bootstrap contents: %s", + bootstrap_contents.c_str()); + } + // Parse bootstrap. + std::unique_ptr<XdsBootstrap> bootstrap = + XdsBootstrap::Create(bootstrap_contents, error); + if (*error != GRPC_ERROR_NONE) return nullptr; + // Instantiate XdsClient. + xds_client = + MakeRefCounted<XdsClient>(std::move(bootstrap), g_channel_args); + g_xds_client = xds_client.get(); } - auto xds_client = MakeRefCounted<XdsClient>(error); - g_xds_client = xds_client.get(); return xds_client; } @@ -2232,6 +2451,66 @@ void UnsetGlobalXdsClientForTest() { g_xds_client = nullptr; } +void SetXdsFallbackBootstrapConfig(const char* config) { + MutexLock lock(g_mu); + gpr_free(g_fallback_bootstrap_config); + g_fallback_bootstrap_config = gpr_strdup(config); +} + } // namespace internal +// +// embedding XdsClient in channel args +// + +#define GRPC_ARG_XDS_CLIENT "grpc.internal.xds_client" + +namespace { + +void* XdsClientArgCopy(void* p) { + XdsClient* xds_client = static_cast<XdsClient*>(p); + xds_client->Ref(DEBUG_LOCATION, "channel arg").release(); + return p; +} + +void XdsClientArgDestroy(void* p) { + XdsClient* xds_client = static_cast<XdsClient*>(p); + xds_client->Unref(DEBUG_LOCATION, "channel arg"); +} + +int XdsClientArgCmp(void* p, void* q) { return GPR_ICMP(p, q); } + +const grpc_arg_pointer_vtable kXdsClientArgVtable = { + XdsClientArgCopy, XdsClientArgDestroy, XdsClientArgCmp}; + +} // namespace + +grpc_arg XdsClient::MakeChannelArg() const { + return grpc_channel_arg_pointer_create(const_cast<char*>(GRPC_ARG_XDS_CLIENT), + const_cast<XdsClient*>(this), + &kXdsClientArgVtable); +} + +RefCountedPtr<XdsClient> XdsClient::GetFromChannelArgs( + const grpc_channel_args& args) { + XdsClient* xds_client = + grpc_channel_args_find_pointer<XdsClient>(&args, GRPC_ARG_XDS_CLIENT); + if (xds_client == nullptr) return nullptr; + return xds_client->Ref(DEBUG_LOCATION, "GetFromChannelArgs"); +} + } // namespace grpc_core + +// The returned bytes may contain NULL(0), so we can't use c-string. +grpc_slice grpc_dump_xds_configs() { + grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; + grpc_core::ExecCtx exec_ctx; + grpc_error_handle error = GRPC_ERROR_NONE; + auto xds_client = grpc_core::XdsClient::GetOrCreate(nullptr, &error); + if (error != GRPC_ERROR_NONE) { + // If we isn't using xDS, just return an empty string. + GRPC_ERROR_UNREF(error); + return grpc_empty_slice(); + } + return grpc_slice_from_cpp_string(xds_client->DumpClientConfigBinary()); +} diff --git a/grpc/src/core/ext/xds/xds_client.h b/grpc/src/core/ext/xds/xds_client.h index f1c64675..0ee84e78 100644 --- a/grpc/src/core/ext/xds/xds_client.h +++ b/grpc/src/core/ext/xds/xds_client.h @@ -48,7 +48,7 @@ class XdsClient : public DualRefCounted<XdsClient> { public: virtual ~ListenerWatcherInterface() = default; virtual void OnListenerChanged(XdsApi::LdsUpdate listener) = 0; - virtual void OnError(grpc_error* error) = 0; + virtual void OnError(grpc_error_handle error) = 0; virtual void OnResourceDoesNotExist() = 0; }; @@ -57,7 +57,7 @@ class XdsClient : public DualRefCounted<XdsClient> { public: virtual ~RouteConfigWatcherInterface() = default; virtual void OnRouteConfigChanged(XdsApi::RdsUpdate route_config) = 0; - virtual void OnError(grpc_error* error) = 0; + virtual void OnError(grpc_error_handle error) = 0; virtual void OnResourceDoesNotExist() = 0; }; @@ -66,7 +66,7 @@ class XdsClient : public DualRefCounted<XdsClient> { public: virtual ~ClusterWatcherInterface() = default; virtual void OnClusterChanged(XdsApi::CdsUpdate cluster_data) = 0; - virtual void OnError(grpc_error* error) = 0; + virtual void OnError(grpc_error_handle error) = 0; virtual void OnResourceDoesNotExist() = 0; }; @@ -75,19 +75,27 @@ class XdsClient : public DualRefCounted<XdsClient> { public: virtual ~EndpointWatcherInterface() = default; virtual void OnEndpointChanged(XdsApi::EdsUpdate update) = 0; - virtual void OnError(grpc_error* error) = 0; + virtual void OnError(grpc_error_handle error) = 0; virtual void OnResourceDoesNotExist() = 0; }; // Factory function to get or create the global XdsClient instance. // If *error is not GRPC_ERROR_NONE upon return, then there was // an error initializing the client. - static RefCountedPtr<XdsClient> GetOrCreate(grpc_error** error); + static RefCountedPtr<XdsClient> GetOrCreate(const grpc_channel_args* args, + grpc_error_handle* error); - // Callers should not instantiate directly. Use GetOrCreate() instead. - explicit XdsClient(grpc_error** error); + // Most callers should not instantiate directly. Use GetOrCreate() instead. + XdsClient(std::unique_ptr<XdsBootstrap> bootstrap, + const grpc_channel_args* args); ~XdsClient() override; + const XdsBootstrap& bootstrap() const { + // bootstrap_ is guaranteed to be non-null since XdsClient::GetOrCreate() + // would return a null object if bootstrap_ was null. + return *bootstrap_; + } + CertificateProviderStore& certificate_provider_store() { return *certificate_provider_store_; } @@ -185,6 +193,20 @@ class XdsClient : public DualRefCounted<XdsClient> { // Resets connection backoff state. void ResetBackoff(); + // Dumps the active xDS config in JSON format. + // Individual xDS resource is encoded as envoy.admin.v3.*ConfigDump. Returns + // envoy.service.status.v3.ClientConfig which also includes the config + // status (e.g., CLIENT_REQUESTED, CLIENT_ACKED, CLIENT_NACKED). + // + // Expected to be invoked by wrapper languages in their CSDS service + // implementation. + std::string DumpClientConfigBinary(); + + // Helpers for encoding the XdsClient object in channel args. + grpc_arg MakeChannelArg() const; + static RefCountedPtr<XdsClient> GetFromChannelArgs( + const grpc_channel_args& args); + private: // Contains a channel to the xds server and all the data related to the // channel. Holds a ref to the xds client object. @@ -215,14 +237,17 @@ class XdsClient : public DualRefCounted<XdsClient> { void MaybeStartLrsCall(); void StopLrsCall(); + bool HasAdsCall() const; bool HasActiveAdsCall() const; void StartConnectivityWatchLocked(); void CancelConnectivityWatchLocked(); - void Subscribe(const std::string& type_url, const std::string& name); - void Unsubscribe(const std::string& type_url, const std::string& name, - bool delay_unsubscription); + void SubscribeLocked(const std::string& type_url, const std::string& name) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + void UnsubscribeLocked(const std::string& type_url, const std::string& name, + bool delay_unsubscription) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); private: class StateWatcher; @@ -248,6 +273,7 @@ class XdsClient : public DualRefCounted<XdsClient> { watchers; // The latest data seen from LDS. absl::optional<XdsApi::LdsUpdate> update; + XdsApi::ResourceMetadata meta; }; struct RouteConfigState { @@ -256,6 +282,7 @@ class XdsClient : public DualRefCounted<XdsClient> { watchers; // The latest data seen from RDS. absl::optional<XdsApi::RdsUpdate> update; + XdsApi::ResourceMetadata meta; }; struct ClusterState { @@ -263,6 +290,7 @@ class XdsClient : public DualRefCounted<XdsClient> { watchers; // The latest data seen from CDS. absl::optional<XdsApi::CdsUpdate> update; + XdsApi::ResourceMetadata meta; }; struct EndpointState { @@ -271,6 +299,7 @@ class XdsClient : public DualRefCounted<XdsClient> { watchers; // The latest data seen from EDS. absl::optional<XdsApi::EdsUpdate> update; + XdsApi::ResourceMetadata meta; }; struct LoadReportState { @@ -288,49 +317,63 @@ class XdsClient : public DualRefCounted<XdsClient> { }; // Sends an error notification to all watchers. - void NotifyOnErrorLocked(grpc_error* error); + void NotifyOnErrorLocked(grpc_error_handle error) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); XdsApi::ClusterLoadReportMap BuildLoadReportSnapshotLocked( - bool send_all_clusters, const std::set<std::string>& clusters); + bool send_all_clusters, const std::set<std::string>& clusters) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + + void UpdateResourceMetadataWithFailedParseResultLocked( + grpc_millis update_time, const XdsApi::AdsParseResult& result) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + std::unique_ptr<XdsBootstrap> bootstrap_; + grpc_channel_args* args_; const grpc_millis request_timeout_; grpc_pollset_set* interested_parties_; - std::unique_ptr<XdsBootstrap> bootstrap_; OrphanablePtr<CertificateProviderStore> certificate_provider_store_; XdsApi api_; Mutex mu_; // The channel for communicating with the xds server. - OrphanablePtr<ChannelState> chand_; + OrphanablePtr<ChannelState> chand_ ABSL_GUARDED_BY(mu_); // One entry for each watched LDS resource. - std::map<std::string /*listener_name*/, ListenerState> listener_map_; + std::map<std::string /*listener_name*/, ListenerState> listener_map_ + ABSL_GUARDED_BY(mu_); // One entry for each watched RDS resource. std::map<std::string /*route_config_name*/, RouteConfigState> - route_config_map_; + route_config_map_ ABSL_GUARDED_BY(mu_); // One entry for each watched CDS resource. - std::map<std::string /*cluster_name*/, ClusterState> cluster_map_; + std::map<std::string /*cluster_name*/, ClusterState> cluster_map_ + ABSL_GUARDED_BY(mu_); // One entry for each watched EDS resource. - std::map<std::string /*eds_service_name*/, EndpointState> endpoint_map_; + std::map<std::string /*eds_service_name*/, EndpointState> endpoint_map_ + ABSL_GUARDED_BY(mu_); // Load report data. std::map< std::pair<std::string /*cluster_name*/, std::string /*eds_service_name*/>, LoadReportState> - load_report_map_; + load_report_map_ ABSL_GUARDED_BY(mu_); // Stores the most recent accepted resource version for each resource type. - std::map<std::string /*type*/, std::string /*version*/> resource_version_map_; + std::map<std::string /*type*/, std::string /*version*/> resource_version_map_ + ABSL_GUARDED_BY(mu_); - bool shutting_down_ = false; + bool shutting_down_ ABSL_GUARDED_BY(mu_) = false; }; namespace internal { void SetXdsChannelArgsForTest(grpc_channel_args* args); void UnsetGlobalXdsClientForTest(); +// Sets bootstrap config to be used when no env var is set. +// Does not take ownership of config. +void SetXdsFallbackBootstrapConfig(const char* config); } // namespace internal } // namespace grpc_core -#endif /* GRPC_CORE_EXT_XDS_XDS_CLIENT_H */ +#endif // GRPC_CORE_EXT_XDS_XDS_CLIENT_H diff --git a/grpc/src/core/ext/xds/xds_client_stats.cc b/grpc/src/core/ext/xds/xds_client_stats.cc index de401a7b..c38ded23 100644 --- a/grpc/src/core/ext/xds/xds_client_stats.cc +++ b/grpc/src/core/ext/xds/xds_client_stats.cc @@ -137,7 +137,8 @@ XdsClusterLocalityStats::GetSnapshotAndReset() { // not related to a single reporting interval. total_requests_in_progress_.Load(MemoryOrder::RELAXED), GetAndResetCounter(&total_error_requests_), - GetAndResetCounter(&total_issued_requests_)}; + GetAndResetCounter(&total_issued_requests_), + {}}; MutexLock lock(&backend_metrics_mu_); snapshot.backend_metrics = std::move(backend_metrics_); return snapshot; diff --git a/grpc/src/core/ext/xds/xds_client_stats.h b/grpc/src/core/ext/xds/xds_client_stats.h index 523ef111..b300fc59 100644 --- a/grpc/src/core/ext/xds/xds_client_stats.h +++ b/grpc/src/core/ext/xds/xds_client_stats.h @@ -56,10 +56,10 @@ class XdsLocalityName : public RefCounted<XdsLocalityName> { } }; - XdsLocalityName(std::string region, std::string zone, std::string subzone) + XdsLocalityName(std::string region, std::string zone, std::string sub_zone) : region_(std::move(region)), zone_(std::move(zone)), - sub_zone_(std::move(subzone)) {} + sub_zone_(std::move(sub_zone)) {} bool operator==(const XdsLocalityName& other) const { return region_ == other.region_ && zone_ == other.zone_ && @@ -149,7 +149,7 @@ class XdsClusterDropStats : public RefCounted<XdsClusterDropStats> { // dropped_requests can be accessed by both the picker (from data plane // mutex) and the load reporting thread (from the control plane combiner). Mutex mu_; - CategorizedDropsMap categorized_drops_; + CategorizedDropsMap categorized_drops_ ABSL_GUARDED_BY(mu_); }; // Locality stats for an xds cluster. @@ -231,7 +231,8 @@ class XdsClusterLocalityStats : public RefCounted<XdsClusterLocalityStats> { // call's recv_trailing_metadata (not from the control plane work serializer) // and the load reporting thread (from the control plane work serializer). Mutex backend_metrics_mu_; - std::map<std::string, BackendMetric> backend_metrics_; + std::map<std::string, BackendMetric> backend_metrics_ + ABSL_GUARDED_BY(backend_metrics_mu_); }; } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_http_fault_filter.cc b/grpc/src/core/ext/xds/xds_http_fault_filter.cc new file mode 100644 index 00000000..64b7a2b4 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_http_fault_filter.cc @@ -0,0 +1,226 @@ +// +// Copyright 2021 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include <grpc/support/port_platform.h> + +#include "src/core/ext/xds/xds_http_fault_filter.h" + +#include <grpc/grpc.h> + +#include <string> + +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "envoy/extensions/filters/common/fault/v3/fault.upb.h" +#include "envoy/extensions/filters/http/fault/v3/fault.upb.h" +#include "envoy/extensions/filters/http/fault/v3/fault.upbdefs.h" +#include "envoy/type/v3/percent.upb.h" +#include "google/protobuf/any.upb.h" +#include "google/protobuf/duration.upb.h" +#include "google/protobuf/wrappers.upb.h" +#include "src/core/ext/filters/fault_injection/fault_injection_filter.h" +#include "src/core/ext/xds/xds_http_filters.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/channel/status_util.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/transport/status_conversion.h" +#include "upb/def.h" + +namespace grpc_core { + +const char* kXdsHttpFaultFilterConfigName = + "envoy.extensions.filters.http.fault.v3.HTTPFault"; + +namespace { + +uint32_t GetDenominator(const envoy_type_v3_FractionalPercent* fraction) { + if (fraction != nullptr) { + const auto denominator = + static_cast<envoy_type_v3_FractionalPercent_DenominatorType>( + envoy_type_v3_FractionalPercent_denominator(fraction)); + switch (denominator) { + case envoy_type_v3_FractionalPercent_MILLION: + return 1000000; + case envoy_type_v3_FractionalPercent_TEN_THOUSAND: + return 10000; + case envoy_type_v3_FractionalPercent_HUNDRED: + default: + return 100; + } + } + // Use 100 as the default denominator + return 100; +} + +absl::StatusOr<Json> ParseHttpFaultIntoJson(upb_strview serialized_http_fault, + upb_arena* arena) { + auto* http_fault = envoy_extensions_filters_http_fault_v3_HTTPFault_parse( + serialized_http_fault.data, serialized_http_fault.size, arena); + if (http_fault == nullptr) { + return absl::InvalidArgumentError( + "could not parse fault injection filter config"); + } + // NOTE(lidiz): Here, we are manually translating the upb messages into the + // JSON form of the filter config as part of method config, which will be + // directly used later by service config. In this way, we can validate the + // filter configs, and NACK if needed. It also allows the service config to + // function independently without xDS, but not the other way around. + // NOTE(lidiz): please refer to FaultInjectionPolicy for ground truth + // definitions, located at: + // src/core/ext/filters/fault_injection/service_config_parser.h + Json::Object fault_injection_policy_json; + // Section 1: Parse the abort injection config + const auto* fault_abort = + envoy_extensions_filters_http_fault_v3_HTTPFault_abort(http_fault); + if (fault_abort != nullptr) { + grpc_status_code abort_grpc_status_code = GRPC_STATUS_OK; + // Try if gRPC status code is set first + int abort_grpc_status_code_raw = + envoy_extensions_filters_http_fault_v3_FaultAbort_grpc_status( + fault_abort); + if (abort_grpc_status_code_raw != 0) { + if (!grpc_status_code_from_int(abort_grpc_status_code_raw, + &abort_grpc_status_code)) { + return absl::InvalidArgumentError(absl::StrCat( + "invalid gRPC status code: ", abort_grpc_status_code_raw)); + } + } else { + // if gRPC status code is empty, check http status + int abort_http_status_code = + envoy_extensions_filters_http_fault_v3_FaultAbort_http_status( + fault_abort); + if (abort_http_status_code != 0 and abort_http_status_code != 200) { + abort_grpc_status_code = + grpc_http2_status_to_grpc_status(abort_http_status_code); + } + } + // Set the abort_code, even if it's OK + fault_injection_policy_json["abortCode"] = + grpc_status_code_to_string(abort_grpc_status_code); + // Set the headers if we enabled header abort injection control + if (envoy_extensions_filters_http_fault_v3_FaultAbort_has_header_abort( + fault_abort)) { + fault_injection_policy_json["abortCodeHeader"] = + "x-envoy-fault-abort-grpc-request"; + fault_injection_policy_json["abortPercentageHeader"] = + "x-envoy-fault-abort-percentage"; + } + // Set the fraction percent + auto* percent = + envoy_extensions_filters_http_fault_v3_FaultAbort_percentage( + fault_abort); + fault_injection_policy_json["abortPercentageNumerator"] = + Json(envoy_type_v3_FractionalPercent_numerator(percent)); + fault_injection_policy_json["abortPercentageDenominator"] = + Json(GetDenominator(percent)); + } + // Section 2: Parse the delay injection config + const auto* fault_delay = + envoy_extensions_filters_http_fault_v3_HTTPFault_delay(http_fault); + if (fault_delay != nullptr) { + // Parse the delay duration + const auto* delay_duration = + envoy_extensions_filters_common_fault_v3_FaultDelay_fixed_delay( + fault_delay); + if (delay_duration != nullptr) { + fault_injection_policy_json["delay"] = absl::StrFormat( + "%d.%09ds", google_protobuf_Duration_seconds(delay_duration), + google_protobuf_Duration_nanos(delay_duration)); + } + // Set the headers if we enabled header delay injection control + if (envoy_extensions_filters_common_fault_v3_FaultDelay_has_header_delay( + fault_delay)) { + fault_injection_policy_json["delayHeader"] = + "x-envoy-fault-delay-request"; + fault_injection_policy_json["delayPercentageHeader"] = + "x-envoy-fault-delay-request-percentage"; + } + // Set the fraction percent + auto* percent = + envoy_extensions_filters_common_fault_v3_FaultDelay_percentage( + fault_delay); + fault_injection_policy_json["delayPercentageNumerator"] = + Json(envoy_type_v3_FractionalPercent_numerator(percent)); + fault_injection_policy_json["delayPercentageDenominator"] = + Json(GetDenominator(percent)); + } + // Section 3: Parse the maximum active faults + const auto* max_fault_wrapper = + envoy_extensions_filters_http_fault_v3_HTTPFault_max_active_faults( + http_fault); + if (max_fault_wrapper != nullptr) { + fault_injection_policy_json["maxFaults"] = + google_protobuf_UInt32Value_value(max_fault_wrapper); + } + return fault_injection_policy_json; +} + +} // namespace + +void XdsHttpFaultFilter::PopulateSymtab(upb_symtab* symtab) const { + envoy_extensions_filters_http_fault_v3_HTTPFault_getmsgdef(symtab); +} + +absl::StatusOr<XdsHttpFilterImpl::FilterConfig> +XdsHttpFaultFilter::GenerateFilterConfig(upb_strview serialized_filter_config, + upb_arena* arena) const { + absl::StatusOr<Json> parse_result = + ParseHttpFaultIntoJson(serialized_filter_config, arena); + if (!parse_result.ok()) { + return parse_result.status(); + } + return FilterConfig{kXdsHttpFaultFilterConfigName, std::move(*parse_result)}; +} + +absl::StatusOr<XdsHttpFilterImpl::FilterConfig> +XdsHttpFaultFilter::GenerateFilterConfigOverride( + upb_strview serialized_filter_config, upb_arena* arena) const { + // HTTPFault filter has the same message type in HTTP connection manager's + // filter config and in overriding filter config field. + return GenerateFilterConfig(serialized_filter_config, arena); +} + +const grpc_channel_filter* XdsHttpFaultFilter::channel_filter() const { + return &FaultInjectionFilterVtable; +} + +grpc_channel_args* XdsHttpFaultFilter::ModifyChannelArgs( + grpc_channel_args* args) const { + grpc_arg args_to_add = grpc_channel_arg_integer_create( + const_cast<char*>(GRPC_ARG_PARSE_FAULT_INJECTION_METHOD_CONFIG), 1); + grpc_channel_args* new_args = + grpc_channel_args_copy_and_add(args, &args_to_add, 1); + // Since this function takes the ownership of the channel args, it needs to + // deallocate the old ones to prevent leak. + grpc_channel_args_destroy(args); + return new_args; +} + +absl::StatusOr<XdsHttpFilterImpl::ServiceConfigJsonEntry> +XdsHttpFaultFilter::GenerateServiceConfig( + const FilterConfig& hcm_filter_config, + const FilterConfig* filter_config_override) const { + Json policy_json = filter_config_override != nullptr + ? filter_config_override->config + : hcm_filter_config.config; + // The policy JSON may be empty, that's allowed. + return ServiceConfigJsonEntry{"faultInjectionPolicy", policy_json.Dump()}; +} + +} // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_http_fault_filter.h b/grpc/src/core/ext/xds/xds_http_fault_filter.h new file mode 100644 index 00000000..60c49524 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_http_fault_filter.h @@ -0,0 +1,63 @@ +// +// Copyright 2021 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_CORE_EXT_XDS_XDS_HTTP_FAULT_FILTER_H +#define GRPC_CORE_EXT_XDS_XDS_HTTP_FAULT_FILTER_H + +#include <grpc/support/port_platform.h> + +#include <grpc/grpc.h> + +#include "absl/status/statusor.h" +#include "src/core/ext/xds/xds_http_filters.h" +#include "upb/def.h" + +namespace grpc_core { + +extern const char* kXdsHttpFaultFilterConfigName; + +class XdsHttpFaultFilter : public XdsHttpFilterImpl { + public: + // Overrides the PopulateSymtab method + void PopulateSymtab(upb_symtab* symtab) const override; + + // Overrides the GenerateFilterConfig method + absl::StatusOr<FilterConfig> GenerateFilterConfig( + upb_strview serialized_filter_config, upb_arena* arena) const override; + + // Overrides the GenerateFilterConfigOverride method + absl::StatusOr<FilterConfig> GenerateFilterConfigOverride( + upb_strview serialized_filter_config, upb_arena* arena) const override; + + // Overrides the channel_filter method + const grpc_channel_filter* channel_filter() const override; + + // Overrides the ModifyChannelArgs method + grpc_channel_args* ModifyChannelArgs(grpc_channel_args* args) const override; + + // Overrides the GenerateServiceConfig method + absl::StatusOr<ServiceConfigJsonEntry> GenerateServiceConfig( + const FilterConfig& hcm_filter_config, + const FilterConfig* filter_config_override) const override; + + bool IsSupportedOnClients() const override { return true; } + + bool IsSupportedOnServers() const override { return false; } +}; + +} // namespace grpc_core + +#endif /* GRPC_CORE_EXT_XDS_XDS_HTTP_FAULT_FILTER_H */ diff --git a/grpc/src/core/ext/xds/xds_http_filters.cc b/grpc/src/core/ext/xds/xds_http_filters.cc new file mode 100644 index 00000000..9bd4858b --- /dev/null +++ b/grpc/src/core/ext/xds/xds_http_filters.cc @@ -0,0 +1,114 @@ +// +// Copyright 2021 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include <grpc/support/port_platform.h> + +#include "src/core/ext/xds/xds_http_filters.h" + +#include "envoy/extensions/filters/http/router/v3/router.upb.h" +#include "envoy/extensions/filters/http/router/v3/router.upbdefs.h" +#include "src/core/ext/xds/xds_http_fault_filter.h" + +namespace grpc_core { + +const char* kXdsHttpRouterFilterConfigName = + "envoy.extensions.filters.http.router.v3.Router"; + +namespace { + +class XdsHttpRouterFilter : public XdsHttpFilterImpl { + public: + void PopulateSymtab(upb_symtab* symtab) const override { + envoy_extensions_filters_http_router_v3_Router_getmsgdef(symtab); + } + + absl::StatusOr<FilterConfig> GenerateFilterConfig( + upb_strview serialized_filter_config, upb_arena* arena) const override { + if (envoy_extensions_filters_http_router_v3_Router_parse( + serialized_filter_config.data, serialized_filter_config.size, + arena) == nullptr) { + return absl::InvalidArgumentError("could not parse router filter config"); + } + return FilterConfig{kXdsHttpRouterFilterConfigName, Json()}; + } + + absl::StatusOr<FilterConfig> GenerateFilterConfigOverride( + upb_strview /*serialized_filter_config*/, + upb_arena* /*arena*/) const override { + return absl::InvalidArgumentError( + "router filter does not support config override"); + } + + // No-op -- this filter is special-cased by the xds resolver. + const grpc_channel_filter* channel_filter() const override { return nullptr; } + + // No-op -- this filter is special-cased by the xds resolver. + absl::StatusOr<ServiceConfigJsonEntry> GenerateServiceConfig( + const FilterConfig& /*hcm_filter_config*/, + const FilterConfig* /*filter_config_override*/) const override { + return absl::UnimplementedError("router filter should never be called"); + } + + bool IsSupportedOnClients() const override { return true; } + + bool IsSupportedOnServers() const override { return true; } +}; + +using FilterOwnerList = std::vector<std::unique_ptr<XdsHttpFilterImpl>>; +using FilterRegistryMap = std::map<absl::string_view, XdsHttpFilterImpl*>; + +FilterOwnerList* g_filters = nullptr; +FilterRegistryMap* g_filter_registry = nullptr; + +} // namespace + +void XdsHttpFilterRegistry::RegisterFilter( + std::unique_ptr<XdsHttpFilterImpl> filter, + const std::set<absl::string_view>& config_proto_type_names) { + for (auto config_proto_type_name : config_proto_type_names) { + (*g_filter_registry)[config_proto_type_name] = filter.get(); + } + g_filters->push_back(std::move(filter)); +} + +const XdsHttpFilterImpl* XdsHttpFilterRegistry::GetFilterForType( + absl::string_view proto_type_name) { + auto it = g_filter_registry->find(proto_type_name); + if (it == g_filter_registry->end()) return nullptr; + return it->second; +} + +void XdsHttpFilterRegistry::PopulateSymtab(upb_symtab* symtab) { + for (const auto& filter : *g_filters) { + filter->PopulateSymtab(symtab); + } +} + +void XdsHttpFilterRegistry::Init() { + g_filters = new FilterOwnerList; + g_filter_registry = new FilterRegistryMap; + RegisterFilter(absl::make_unique<XdsHttpRouterFilter>(), + {kXdsHttpRouterFilterConfigName}); + RegisterFilter(absl::make_unique<XdsHttpFaultFilter>(), + {kXdsHttpFaultFilterConfigName}); +} + +void XdsHttpFilterRegistry::Shutdown() { + delete g_filter_registry; + delete g_filters; +} + +} // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_http_filters.h b/grpc/src/core/ext/xds/xds_http_filters.h new file mode 100644 index 00000000..33241968 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_http_filters.h @@ -0,0 +1,130 @@ +// +// Copyright 2021 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_CORE_EXT_XDS_XDS_HTTP_FILTERS_H +#define GRPC_CORE_EXT_XDS_XDS_HTTP_FILTERS_H + +#include <grpc/support/port_platform.h> + +#include <memory> +#include <set> +#include <string> + +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "google/protobuf/any.upb.h" +#include "upb/def.h" + +#include <grpc/grpc.h> + +#include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/json/json.h" + +namespace grpc_core { + +extern const char* kXdsHttpRouterFilterConfigName; + +class XdsHttpFilterImpl { + public: + struct FilterConfig { + absl::string_view config_proto_type_name; + Json config; + + bool operator==(const FilterConfig& other) const { + return config_proto_type_name == other.config_proto_type_name && + config == other.config; + } + std::string ToString() const { + return absl::StrCat("{config_proto_type_name=", config_proto_type_name, + " config=", config.Dump(), "}"); + } + }; + + // Service config data for the filter, returned by GenerateServiceConfig(). + struct ServiceConfigJsonEntry { + // The top-level field name in the method config. + // Filter implementations should use their primary config proto type + // name for this. + // The value of this field in the method config will be a JSON array, + // which will be populated with the elements returned by each filter + // instance. + std::string service_config_field_name; + // The element to add to the JSON array. + std::string element; + }; + + virtual ~XdsHttpFilterImpl() = default; + + // Loads the proto message into the upb symtab. + virtual void PopulateSymtab(upb_symtab* symtab) const = 0; + + // Generates a Config from the xDS filter config proto. + // Used for the top-level config in the HCM HTTP filter list. + virtual absl::StatusOr<FilterConfig> GenerateFilterConfig( + upb_strview serialized_filter_config, upb_arena* arena) const = 0; + + // Generates a Config from the xDS filter config proto. + // Used for the typed_per_filter_config override in VirtualHost and Route. + virtual absl::StatusOr<FilterConfig> GenerateFilterConfigOverride( + upb_strview serialized_filter_config, upb_arena* arena) const = 0; + + // C-core channel filter implementation. + virtual const grpc_channel_filter* channel_filter() const = 0; + + // Modifies channel args that may affect service config parsing (not + // visible to the channel as a whole). + // Takes ownership of args. Caller takes ownership of return value. + virtual grpc_channel_args* ModifyChannelArgs(grpc_channel_args* args) const { + return args; + } + + // Function to convert the Configs into a JSON string to be added to the + // per-method part of the service config. + // The hcm_filter_config comes from the HttpConnectionManager config. + // The filter_config_override comes from the first of the ClusterWeight, + // Route, or VirtualHost entries that it is found in, or null if + // there is no override in any of those locations. + virtual absl::StatusOr<ServiceConfigJsonEntry> GenerateServiceConfig( + const FilterConfig& hcm_filter_config, + const FilterConfig* filter_config_override) const = 0; + + // Returns true if the filter is supported on clients; false otherwise + virtual bool IsSupportedOnClients() const = 0; + + // Returns true if the filter is supported on servers; false otherwise + virtual bool IsSupportedOnServers() const = 0; +}; + +class XdsHttpFilterRegistry { + public: + static void RegisterFilter( + std::unique_ptr<XdsHttpFilterImpl> filter, + const std::set<absl::string_view>& config_proto_type_names); + + static const XdsHttpFilterImpl* GetFilterForType( + absl::string_view proto_type_name); + + static void PopulateSymtab(upb_symtab* symtab); + + // Global init and shutdown. + static void Init(); + static void Shutdown(); +}; + +} // namespace grpc_core + +#endif /* GRPC_CORE_EXT_XDS_XDS_HTTP_FILTERS_H */ diff --git a/grpc/src/core/ext/xds/xds_server_config_fetcher.cc b/grpc/src/core/ext/xds/xds_server_config_fetcher.cc index 5c5e8ee2..962731f9 100644 --- a/grpc/src/core/ext/xds/xds_server_config_fetcher.cc +++ b/grpc/src/core/ext/xds/xds_server_config_fetcher.cc @@ -18,32 +18,354 @@ #include <grpc/support/port_platform.h> +#include "absl/strings/str_replace.h" + +#include "src/core/ext/xds/xds_certificate_provider.h" #include "src/core/ext/xds/xds_client.h" +#include "src/core/lib/address_utils/sockaddr_utils.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/host_port.h" +#include "src/core/lib/iomgr/sockaddr.h" +#include "src/core/lib/iomgr/socket_utils.h" +#include "src/core/lib/security/credentials/xds/xds_credentials.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/server.h" +#include "src/core/lib/uri/uri_parser.h" namespace grpc_core { + +TraceFlag grpc_xds_server_config_fetcher_trace(false, + "xds_server_config_fetcher"); + namespace { +class FilterChainMatchManager + : public grpc_server_config_fetcher::ConnectionManager { + public: + FilterChainMatchManager( + RefCountedPtr<XdsClient> xds_client, + XdsApi::LdsUpdate::FilterChainMap filter_chain_map, + absl::optional<XdsApi::LdsUpdate::FilterChainData> default_filter_chain) + : xds_client_(xds_client), + filter_chain_map_(std::move(filter_chain_map)), + default_filter_chain_(std::move(default_filter_chain)) {} + + absl::StatusOr<grpc_channel_args*> UpdateChannelArgsForConnection( + grpc_channel_args* args, grpc_endpoint* tcp) override; + + const XdsApi::LdsUpdate::FilterChainMap& filter_chain_map() const { + return filter_chain_map_; + } + + const absl::optional<XdsApi::LdsUpdate::FilterChainData>& + default_filter_chain() const { + return default_filter_chain_; + } + + private: + struct CertificateProviders { + // We need to save our own refs to the root and instance certificate + // providers since the xds certificate provider just stores a ref to their + // distributors. + RefCountedPtr<grpc_tls_certificate_provider> root; + RefCountedPtr<grpc_tls_certificate_provider> instance; + RefCountedPtr<XdsCertificateProvider> xds; + }; + + absl::StatusOr<RefCountedPtr<XdsCertificateProvider>> + CreateOrGetXdsCertificateProviderFromFilterChainData( + const XdsApi::LdsUpdate::FilterChainData* filter_chain); + + const RefCountedPtr<XdsClient> xds_client_; + const XdsApi::LdsUpdate::FilterChainMap filter_chain_map_; + const absl::optional<XdsApi::LdsUpdate::FilterChainData> + default_filter_chain_; + Mutex mu_; + std::map<const XdsApi::LdsUpdate::FilterChainData*, CertificateProviders> + certificate_providers_map_ ABSL_GUARDED_BY(mu_); +}; + +bool IsLoopbackIp(const grpc_resolved_address* address) { + const grpc_sockaddr* sock_addr = + reinterpret_cast<const grpc_sockaddr*>(&address->addr); + if (sock_addr->sa_family == GRPC_AF_INET) { + const grpc_sockaddr_in* addr4 = + reinterpret_cast<const grpc_sockaddr_in*>(sock_addr); + if (addr4->sin_addr.s_addr == grpc_htonl(INADDR_LOOPBACK)) { + return true; + } + } else if (sock_addr->sa_family == GRPC_AF_INET6) { + const grpc_sockaddr_in6* addr6 = + reinterpret_cast<const grpc_sockaddr_in6*>(sock_addr); + if (memcmp(&addr6->sin6_addr, &in6addr_loopback, + sizeof(in6addr_loopback)) == 0) { + return true; + } + } + return false; +} + +const XdsApi::LdsUpdate::FilterChainData* FindFilterChainDataForSourcePort( + const XdsApi::LdsUpdate::FilterChainMap::SourcePortsMap& source_ports_map, + absl::string_view port_str) { + int port = 0; + if (!absl::SimpleAtoi(port_str, &port)) return nullptr; + auto it = source_ports_map.find(port); + if (it != source_ports_map.end()) { + return it->second.data.get(); + } + // Search for the catch-all port 0 since we didn't get a direct match + it = source_ports_map.find(0); + if (it != source_ports_map.end()) { + return it->second.data.get(); + } + return nullptr; +} + +const XdsApi::LdsUpdate::FilterChainData* FindFilterChainDataForSourceIp( + const XdsApi::LdsUpdate::FilterChainMap::SourceIpVector& source_ip_vector, + const grpc_resolved_address* source_ip, absl::string_view port) { + const XdsApi::LdsUpdate::FilterChainMap::SourceIp* best_match = nullptr; + for (const auto& entry : source_ip_vector) { + // Special case for catch-all + if (!entry.prefix_range.has_value()) { + if (best_match == nullptr) { + best_match = &entry; + } + continue; + } + if (best_match != nullptr && best_match->prefix_range.has_value() && + best_match->prefix_range->prefix_len >= + entry.prefix_range->prefix_len) { + continue; + } + if (grpc_sockaddr_match_subnet(source_ip, &entry.prefix_range->address, + entry.prefix_range->prefix_len)) { + best_match = &entry; + } + } + if (best_match == nullptr) return nullptr; + return FindFilterChainDataForSourcePort(best_match->ports_map, port); +} + +const XdsApi::LdsUpdate::FilterChainData* FindFilterChainDataForSourceType( + const XdsApi::LdsUpdate::FilterChainMap::ConnectionSourceTypesArray& + source_types_array, + grpc_endpoint* tcp, absl::string_view destination_ip) { + auto source_uri = URI::Parse(grpc_endpoint_get_peer(tcp)); + if (!source_uri.ok() || + (source_uri->scheme() != "ipv4" && source_uri->scheme() != "ipv6")) { + return nullptr; + } + std::string host; + std::string port; + if (!SplitHostPort(source_uri->path(), &host, &port)) { + return nullptr; + } + grpc_resolved_address source_addr; + grpc_error_handle error = grpc_string_to_sockaddr( + &source_addr, host.c_str(), 0 /* port doesn't matter here */); + if (error != GRPC_ERROR_NONE) { + gpr_log(GPR_DEBUG, "Could not parse string to socket address: %s", + host.c_str()); + GRPC_ERROR_UNREF(error); + return nullptr; + } + // Use kAny only if kSameIporLoopback and kExternal are empty + if (source_types_array[static_cast<int>( + XdsApi::LdsUpdate::FilterChainMap:: + ConnectionSourceType::kSameIpOrLoopback)] + .empty() && + source_types_array[static_cast<int>(XdsApi::LdsUpdate::FilterChainMap:: + ConnectionSourceType::kExternal)] + .empty()) { + return FindFilterChainDataForSourceIp( + source_types_array[static_cast<int>( + XdsApi::LdsUpdate::FilterChainMap::ConnectionSourceType::kAny)], + &source_addr, port); + } + if (IsLoopbackIp(&source_addr) || host == destination_ip) { + return FindFilterChainDataForSourceIp( + source_types_array[static_cast<int>( + XdsApi::LdsUpdate::FilterChainMap::ConnectionSourceType:: + kSameIpOrLoopback)], + &source_addr, port); + } else { + return FindFilterChainDataForSourceIp( + source_types_array[static_cast<int>( + XdsApi::LdsUpdate::FilterChainMap::ConnectionSourceType:: + kExternal)], + &source_addr, port); + } +} + +const XdsApi::LdsUpdate::FilterChainData* FindFilterChainDataForDestinationIp( + const XdsApi::LdsUpdate::FilterChainMap::DestinationIpVector + destination_ip_vector, + grpc_endpoint* tcp) { + auto destination_uri = URI::Parse(grpc_endpoint_get_local_address(tcp)); + if (!destination_uri.ok() || (destination_uri->scheme() != "ipv4" && + destination_uri->scheme() != "ipv6")) { + return nullptr; + } + std::string host; + std::string port; + if (!SplitHostPort(destination_uri->path(), &host, &port)) { + return nullptr; + } + grpc_resolved_address destination_addr; + grpc_error_handle error = grpc_string_to_sockaddr( + &destination_addr, host.c_str(), 0 /* port doesn't matter here */); + if (error != GRPC_ERROR_NONE) { + gpr_log(GPR_DEBUG, "Could not parse string to socket address: %s", + host.c_str()); + GRPC_ERROR_UNREF(error); + return nullptr; + } + const XdsApi::LdsUpdate::FilterChainMap::DestinationIp* best_match = nullptr; + for (const auto& entry : destination_ip_vector) { + // Special case for catch-all + if (!entry.prefix_range.has_value()) { + if (best_match == nullptr) { + best_match = &entry; + } + continue; + } + if (best_match != nullptr && best_match->prefix_range.has_value() && + best_match->prefix_range->prefix_len >= + entry.prefix_range->prefix_len) { + continue; + } + if (grpc_sockaddr_match_subnet(&destination_addr, + &entry.prefix_range->address, + entry.prefix_range->prefix_len)) { + best_match = &entry; + } + } + if (best_match == nullptr) return nullptr; + return FindFilterChainDataForSourceType(best_match->source_types_array, tcp, + host); +} + +absl::StatusOr<RefCountedPtr<XdsCertificateProvider>> +FilterChainMatchManager::CreateOrGetXdsCertificateProviderFromFilterChainData( + const XdsApi::LdsUpdate::FilterChainData* filter_chain) { + MutexLock lock(&mu_); + auto it = certificate_providers_map_.find(filter_chain); + if (it != certificate_providers_map_.end()) { + return it->second.xds; + } + CertificateProviders certificate_providers; + // Configure root cert. + absl::string_view root_provider_instance_name = + filter_chain->downstream_tls_context.common_tls_context + .combined_validation_context + .validation_context_certificate_provider_instance.instance_name; + absl::string_view root_provider_cert_name = + filter_chain->downstream_tls_context.common_tls_context + .combined_validation_context + .validation_context_certificate_provider_instance.certificate_name; + if (!root_provider_instance_name.empty()) { + certificate_providers.root = + xds_client_->certificate_provider_store() + .CreateOrGetCertificateProvider(root_provider_instance_name); + if (certificate_providers.root == nullptr) { + return absl::NotFoundError( + absl::StrCat("Certificate provider instance name: \"", + root_provider_instance_name, "\" not recognized.")); + } + } + // Configure identity cert. + absl::string_view identity_provider_instance_name = + filter_chain->downstream_tls_context.common_tls_context + .tls_certificate_certificate_provider_instance.instance_name; + absl::string_view identity_provider_cert_name = + filter_chain->downstream_tls_context.common_tls_context + .tls_certificate_certificate_provider_instance.certificate_name; + if (!identity_provider_instance_name.empty()) { + certificate_providers.instance = + xds_client_->certificate_provider_store() + .CreateOrGetCertificateProvider(identity_provider_instance_name); + if (certificate_providers.instance == nullptr) { + return absl::NotFoundError( + absl::StrCat("Certificate provider instance name: \"", + identity_provider_instance_name, "\" not recognized.")); + } + } + certificate_providers.xds = MakeRefCounted<XdsCertificateProvider>(); + certificate_providers.xds->UpdateRootCertNameAndDistributor( + "", root_provider_cert_name, + certificate_providers.root == nullptr + ? nullptr + : certificate_providers.root->distributor()); + certificate_providers.xds->UpdateIdentityCertNameAndDistributor( + "", identity_provider_cert_name, + certificate_providers.instance == nullptr + ? nullptr + : certificate_providers.instance->distributor()); + certificate_providers.xds->UpdateRequireClientCertificate( + "", filter_chain->downstream_tls_context.require_client_certificate); + auto xds_certificate_provider = certificate_providers.xds; + certificate_providers_map_.emplace(filter_chain, + std::move(certificate_providers)); + return xds_certificate_provider; +} + +absl::StatusOr<grpc_channel_args*> +FilterChainMatchManager::UpdateChannelArgsForConnection(grpc_channel_args* args, + grpc_endpoint* tcp) { + const auto* filter_chain = FindFilterChainDataForDestinationIp( + filter_chain_map_.destination_ip_vector, tcp); + if (filter_chain == nullptr && default_filter_chain_.has_value()) { + filter_chain = &default_filter_chain_.value(); + } + if (filter_chain == nullptr) { + grpc_channel_args_destroy(args); + return absl::UnavailableError("No matching filter chain found"); + } + // Nothing to update if credentials are not xDS. + grpc_server_credentials* server_creds = + grpc_find_server_credentials_in_args(args); + if (server_creds == nullptr || server_creds->type() != kCredentialsTypeXds) { + return args; + } + absl::StatusOr<RefCountedPtr<XdsCertificateProvider>> result = + CreateOrGetXdsCertificateProviderFromFilterChainData(filter_chain); + if (!result.ok()) { + grpc_channel_args_destroy(args); + return result.status(); + } + RefCountedPtr<XdsCertificateProvider> xds_certificate_provider = + std::move(*result); + GPR_ASSERT(xds_certificate_provider != nullptr); + grpc_arg arg_to_add = xds_certificate_provider->MakeChannelArg(); + grpc_channel_args* updated_args = + grpc_channel_args_copy_and_add(args, &arg_to_add, 1); + grpc_channel_args_destroy(args); + return updated_args; +} + class XdsServerConfigFetcher : public grpc_server_config_fetcher { public: - explicit XdsServerConfigFetcher(RefCountedPtr<XdsClient> xds_client) - : xds_client_(std::move(xds_client)) { + explicit XdsServerConfigFetcher(RefCountedPtr<XdsClient> xds_client, + grpc_server_xds_status_notifier notifier) + : xds_client_(std::move(xds_client)), serving_status_notifier_(notifier) { GPR_ASSERT(xds_client_ != nullptr); } - void StartWatch(std::string listening_address, + void StartWatch(std::string listening_address, grpc_channel_args* args, std::unique_ptr<grpc_server_config_fetcher::WatcherInterface> watcher) override { grpc_server_config_fetcher::WatcherInterface* watcher_ptr = watcher.get(); - auto listener_watcher = - absl::make_unique<ListenerWatcher>(std::move(watcher)); + auto listener_watcher = absl::make_unique<ListenerWatcher>( + std::move(watcher), args, xds_client_, serving_status_notifier_, + listening_address); auto* listener_watcher_ptr = listener_watcher.get(); - // TODO(yashykt): Get the resource name id from bootstrap - xds_client_->WatchListenerData( - absl::StrCat("grpc/server?xds.resource.listening_address=", - listening_address), - std::move(listener_watcher)); + listening_address = absl::StrReplaceAll( + xds_client_->bootstrap().server_listener_resource_name_template(), + {{"%s", listening_address}}); + xds_client_->WatchListenerData(listening_address, + std::move(listener_watcher)); MutexLock lock(&mu_); auto& watcher_state = watchers_[watcher_ptr]; watcher_state.listening_address = listening_address; @@ -73,32 +395,114 @@ class XdsServerConfigFetcher : public grpc_server_config_fetcher { public: explicit ListenerWatcher( std::unique_ptr<grpc_server_config_fetcher::WatcherInterface> - server_config_watcher) - : server_config_watcher_(std::move(server_config_watcher)) {} + server_config_watcher, + grpc_channel_args* args, RefCountedPtr<XdsClient> xds_client, + grpc_server_xds_status_notifier serving_status_notifier, + std::string listening_address) + : server_config_watcher_(std::move(server_config_watcher)), + args_(args), + xds_client_(std::move(xds_client)), + serving_status_notifier_(serving_status_notifier), + listening_address_(std::move(listening_address)) {} + + ~ListenerWatcher() override { grpc_channel_args_destroy(args_); } + + // Deleted due to special handling required for args_. Copy the channel args + // if we ever need these. + ListenerWatcher(const ListenerWatcher&) = delete; + ListenerWatcher& operator=(const ListenerWatcher&) = delete; void OnListenerChanged(XdsApi::LdsUpdate listener) override { - // TODO(yashykt): Construct channel args according to received update - server_config_watcher_->UpdateConfig(nullptr); + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_server_config_fetcher_trace)) { + gpr_log( + GPR_INFO, + "[ListenerWatcher %p] Received LDS update from xds client %p: %s", + this, xds_client_.get(), listener.ToString().c_str()); + } + if (listener.address != listening_address_) { + OnFatalError(absl::FailedPreconditionError( + "Address in LDS update does not match listening address")); + return; + } + if (filter_chain_match_manager_ == nullptr) { + if (serving_status_notifier_.on_serving_status_update != nullptr) { + serving_status_notifier_.on_serving_status_update( + serving_status_notifier_.user_data, listening_address_.c_str(), + GRPC_STATUS_OK, ""); + } else { + gpr_log(GPR_INFO, + "xDS Listener resource obtained; will start serving on %s", + listening_address_.c_str()); + } + } + if (filter_chain_match_manager_ == nullptr || + !(listener.filter_chain_map == + filter_chain_match_manager_->filter_chain_map() && + listener.default_filter_chain == + filter_chain_match_manager_->default_filter_chain())) { + filter_chain_match_manager_ = MakeRefCounted<FilterChainMatchManager>( + xds_client_, std::move(listener.filter_chain_map), + std::move(listener.default_filter_chain)); + server_config_watcher_->UpdateConnectionManager( + filter_chain_match_manager_); + } } - void OnError(grpc_error* error) override { - gpr_log(GPR_ERROR, "ListenerWatcher:%p XdsClient reports error: %s", this, - grpc_error_string(error)); + void OnError(grpc_error_handle error) override { + if (filter_chain_match_manager_ != nullptr) { + gpr_log(GPR_ERROR, + "ListenerWatcher:%p XdsClient reports error: %s for %s; " + "ignoring in favor of existing resource", + this, grpc_error_std_string(error).c_str(), + listening_address_.c_str()); + } else { + if (serving_status_notifier_.on_serving_status_update != nullptr) { + serving_status_notifier_.on_serving_status_update( + serving_status_notifier_.user_data, listening_address_.c_str(), + GRPC_STATUS_UNAVAILABLE, grpc_error_std_string(error).c_str()); + } else { + gpr_log( + GPR_ERROR, + "ListenerWatcher:%p error obtaining xDS Listener resource: %s; " + "not serving on %s", + this, grpc_error_std_string(error).c_str(), + listening_address_.c_str()); + } + } GRPC_ERROR_UNREF(error); - // TODO(yashykt): We might want to bubble this error to the application. + } + + void OnFatalError(absl::Status status) { + gpr_log( + GPR_ERROR, + "ListenerWatcher:%p Encountered fatal error %s; not serving on %s", + this, status.ToString().c_str(), listening_address_.c_str()); + if (filter_chain_match_manager_ != nullptr) { + // The server has started listening already, so we need to gracefully + // stop serving. + server_config_watcher_->StopServing(); + filter_chain_match_manager_.reset(); + } + if (serving_status_notifier_.on_serving_status_update != nullptr) { + serving_status_notifier_.on_serving_status_update( + serving_status_notifier_.user_data, listening_address_.c_str(), + static_cast<grpc_status_code>(status.raw_code()), + std::string(status.message()).c_str()); + } } void OnResourceDoesNotExist() override { - gpr_log(GPR_ERROR, - "ListenerWatcher:%p XdsClient reports requested listener does " - "not exist", - this); - // TODO(yashykt): We might want to bubble this error to the application. + OnFatalError(absl::NotFoundError("Requested listener does not exist")); } private: std::unique_ptr<grpc_server_config_fetcher::WatcherInterface> server_config_watcher_; + grpc_channel_args* args_; + RefCountedPtr<XdsClient> xds_client_; + grpc_server_xds_status_notifier serving_status_notifier_; + std::string listening_address_; + RefCountedPtr<FilterChainMatchManager> filter_chain_match_manager_; }; struct WatcherState { @@ -107,25 +511,36 @@ class XdsServerConfigFetcher : public grpc_server_config_fetcher { }; RefCountedPtr<XdsClient> xds_client_; + grpc_server_xds_status_notifier serving_status_notifier_; Mutex mu_; std::map<grpc_server_config_fetcher::WatcherInterface*, WatcherState> - watchers_; + watchers_ ABSL_GUARDED_BY(mu_); }; } // namespace } // namespace grpc_core -grpc_server_config_fetcher* grpc_server_config_fetcher_xds_create() { +grpc_server_config_fetcher* grpc_server_config_fetcher_xds_create( + grpc_server_xds_status_notifier notifier, const grpc_channel_args* args) { grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; grpc_core::ExecCtx exec_ctx; GRPC_API_TRACE("grpc_server_config_fetcher_xds_create()", 0, ()); - grpc_error* error = GRPC_ERROR_NONE; + grpc_error_handle error = GRPC_ERROR_NONE; grpc_core::RefCountedPtr<grpc_core::XdsClient> xds_client = - grpc_core::XdsClient::GetOrCreate(&error); + grpc_core::XdsClient::GetOrCreate(args, &error); if (error != GRPC_ERROR_NONE) { gpr_log(GPR_ERROR, "Failed to create xds client: %s", - grpc_error_string(error)); + grpc_error_std_string(error).c_str()); + GRPC_ERROR_UNREF(error); + return nullptr; + } + if (xds_client->bootstrap() + .server_listener_resource_name_template() + .empty()) { + gpr_log(GPR_ERROR, + "server_listener_resource_name_template not provided in bootstrap " + "file."); return nullptr; } - return new grpc_core::XdsServerConfigFetcher(std::move(xds_client)); + return new grpc_core::XdsServerConfigFetcher(std::move(xds_client), notifier); } |