diff options
| author | Shuo Wang Hsu <shuohsu@google.com> | 2024-01-02 16:54:20 -0800 |
|---|---|---|
| committer | Shuo Wang Hsu <shuohsu@google.com> | 2024-01-02 16:54:22 -0800 |
| commit | 67249fccf527662d90aed4eed2181fdd4cb682e3 (patch) | |
| tree | 38fcc3ed3fb25375eaf918489ea17913adca8ebe /grpc/src/core/ext/xds | |
| parent | dde934ee51d1a9fa5562284fb0aface52c7adc2e (diff) | |
| download | grpcio-sys-67249fccf527662d90aed4eed2181fdd4cb682e3.tar.gz | |
Upgrade grpcio-sys to 0.13.0+1.56.2-patched
This project was upgraded with external_updater.
Usage: tools/external_updater/updater.sh update rust/crates/grpcio-sys
For more info, check https://cs.android.com/android/platform/superproject/+/main:tools/external_updater/README.md
Test: TreeHugger
Change-Id: Ife139c5ce97cbc54042f0820fe3caee5a7bdf083
Diffstat (limited to 'grpc/src/core/ext/xds')
62 files changed, 7545 insertions, 5719 deletions
diff --git a/grpc/src/core/ext/xds/certificate_provider_factory.h b/grpc/src/core/ext/xds/certificate_provider_factory.h deleted file mode 100644 index e9bba790..00000000 --- a/grpc/src/core/ext/xds/certificate_provider_factory.h +++ /dev/null @@ -1,61 +0,0 @@ -// -// -// Copyright 2020 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// - -#ifndef GRPC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_FACTORY_H -#define GRPC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_FACTORY_H - -#include <grpc/support/port_platform.h> - -#include "src/core/lib/iomgr/error.h" -#include "src/core/lib/json/json.h" -#include "src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h" - -namespace grpc_core { - -// Factories for plugins. Each plugin implementation should create its own -// factory implementation and register an instance with the registry. -class CertificateProviderFactory { - public: - // Interface for configs for CertificateProviders. - class Config : public RefCounted<Config> { - public: - ~Config() override = default; - - // Name of the type of the CertificateProvider. Unique to each type of - // config. - virtual const char* name() const = 0; - - virtual std::string ToString() const = 0; - }; - - virtual ~CertificateProviderFactory() = default; - - // Name of the plugin. - virtual const char* name() const = 0; - - virtual RefCountedPtr<Config> CreateCertificateProviderConfig( - const Json& config_json, grpc_error_handle* error) = 0; - - // Create a CertificateProvider instance from config. - virtual RefCountedPtr<grpc_tls_certificate_provider> - CreateCertificateProvider(RefCountedPtr<Config> config) = 0; -}; - -} // namespace grpc_core - -#endif // GRPC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_FACTORY_H diff --git a/grpc/src/core/ext/xds/certificate_provider_registry.cc b/grpc/src/core/ext/xds/certificate_provider_registry.cc deleted file mode 100644 index 8802f2be..00000000 --- a/grpc/src/core/ext/xds/certificate_provider_registry.cc +++ /dev/null @@ -1,103 +0,0 @@ -// -// -// Copyright 2020 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// - -#include <grpc/support/port_platform.h> - -#include "src/core/ext/xds/certificate_provider_registry.h" - -#include "absl/container/inlined_vector.h" - -namespace grpc_core { - -namespace { - -class RegistryState { - public: - void RegisterCertificateProviderFactory( - std::unique_ptr<CertificateProviderFactory> factory) { - gpr_log(GPR_DEBUG, "registering certificate provider factory for \"%s\"", - factory->name()); - for (size_t i = 0; i < factories_.size(); ++i) { - GPR_ASSERT(strcmp(factories_[i]->name(), factory->name()) != 0); - } - factories_.push_back(std::move(factory)); - } - - CertificateProviderFactory* LookupCertificateProviderFactory( - absl::string_view name) const { - for (size_t i = 0; i < factories_.size(); ++i) { - if (name == factories_[i]->name()) { - return factories_[i].get(); - } - } - return nullptr; - } - - private: - // We currently support 3 factories without doing additional - // allocation. This number could be raised if there is a case where - // more factories are needed and the additional allocations are - // hurting performance (which is unlikely, since these allocations - // only occur at gRPC initialization time). - absl::InlinedVector<std::unique_ptr<CertificateProviderFactory>, 3> - factories_; -}; - -RegistryState* g_state = nullptr; - -} // namespace - -// -// CertificateProviderRegistry -// - -CertificateProviderFactory* -CertificateProviderRegistry::LookupCertificateProviderFactory( - absl::string_view name) { - GPR_ASSERT(g_state != nullptr); - return g_state->LookupCertificateProviderFactory(name); -} - -void CertificateProviderRegistry::InitRegistry() { - if (g_state == nullptr) g_state = new RegistryState(); -} - -void CertificateProviderRegistry::ShutdownRegistry() { - delete g_state; - g_state = nullptr; -} - -void CertificateProviderRegistry::RegisterCertificateProviderFactory( - std::unique_ptr<CertificateProviderFactory> factory) { - InitRegistry(); - g_state->RegisterCertificateProviderFactory(std::move(factory)); -} - -} // namespace grpc_core - -// -// Plugin registration -// - -void grpc_certificate_provider_registry_init() { - grpc_core::CertificateProviderRegistry::InitRegistry(); -} - -void grpc_certificate_provider_registry_shutdown() { - grpc_core::CertificateProviderRegistry::ShutdownRegistry(); -} diff --git a/grpc/src/core/ext/xds/certificate_provider_registry.h b/grpc/src/core/ext/xds/certificate_provider_registry.h deleted file mode 100644 index 38979765..00000000 --- a/grpc/src/core/ext/xds/certificate_provider_registry.h +++ /dev/null @@ -1,57 +0,0 @@ -// -// -// Copyright 2020 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// - -#ifndef GRPC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_REGISTRY_H -#define GRPC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_REGISTRY_H - -#include <grpc/support/port_platform.h> - -#include <string> - -#include "src/core/ext/xds/certificate_provider_factory.h" - -namespace grpc_core { - -// Global registry for all the certificate provider plugins. -class CertificateProviderRegistry { - public: - // Returns the factory for the plugin keyed by name. - static CertificateProviderFactory* LookupCertificateProviderFactory( - absl::string_view name); - - // The following methods are used to create and populate the - // CertificateProviderRegistry. NOT THREAD SAFE -- to be used only during - // global gRPC initialization and shutdown. - - // Global initialization of the registry. - static void InitRegistry(); - - // Global shutdown of the registry. - static void ShutdownRegistry(); - - // Register a provider with the registry. Can only be called after calling - // InitRegistry(). The key of the factory is extracted from factory - // parameter with method CertificateProviderFactory::name. If the same key - // is registered twice, an exception is raised. - static void RegisterCertificateProviderFactory( - std::unique_ptr<CertificateProviderFactory> factory); -}; - -} // namespace grpc_core - -#endif // GRPC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_REGISTRY_H diff --git a/grpc/src/core/ext/xds/certificate_provider_store.cc b/grpc/src/core/ext/xds/certificate_provider_store.cc index 83001972..b4a13067 100644 --- a/grpc/src/core/ext/xds/certificate_provider_store.cc +++ b/grpc/src/core/ext/xds/certificate_provider_store.cc @@ -20,16 +20,73 @@ #include "src/core/ext/xds/certificate_provider_store.h" -#include "src/core/ext/xds/certificate_provider_registry.h" +#include "absl/strings/str_cat.h" + +#include <grpc/support/json.h> +#include <grpc/support/log.h> + +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/security/certificate_provider/certificate_provider_registry.h" namespace grpc_core { // +// CertificateProviderStore::PluginDefinition +// + +const JsonLoaderInterface* +CertificateProviderStore::PluginDefinition::JsonLoader(const JsonArgs&) { + static const auto* loader = + JsonObjectLoader<PluginDefinition>() + .Field("plugin_name", &PluginDefinition::plugin_name) + .Finish(); + return loader; +} + +void CertificateProviderStore::PluginDefinition::JsonPostLoad( + const Json& json, const JsonArgs& args, ValidationErrors* errors) { + // Check that plugin is supported. + CertificateProviderFactory* factory = nullptr; + if (!plugin_name.empty()) { + ValidationErrors::ScopedField field(errors, ".plugin_name"); + factory = CoreConfiguration::Get() + .certificate_provider_registry() + .LookupCertificateProviderFactory(plugin_name); + if (factory == nullptr) { + errors->AddError(absl::StrCat("Unrecognized plugin name: ", plugin_name)); + return; // No point checking config. + } + } + // Parse the config field. + { + ValidationErrors::ScopedField field(errors, ".config"); + auto it = json.object().find("config"); + // The config field is optional; if not present, we use an empty JSON + // object. + Json::Object config_json; + if (it != json.object().end()) { + if (it->second.type() != Json::Type::kObject) { + errors->AddError("is not an object"); + return; // No point parsing config. + } else { + config_json = it->second.object(); + } + } + if (factory == nullptr) return; + // Use plugin to validate and parse config. + config = factory->CreateCertificateProviderConfig( + Json::FromObject(std::move(config_json)), args, errors); + } +} + +// // CertificateProviderStore::CertificateProviderWrapper // -const char* CertificateProviderStore::CertificateProviderWrapper::type() const { - return "Wrapper"; +UniqueTypeName CertificateProviderStore::CertificateProviderWrapper::type() + const { + static UniqueTypeName::Factory kFactory("Wrapper"); + return kFactory.Create(); } // If a certificate provider is created, the CertificateProviderStore @@ -66,8 +123,10 @@ CertificateProviderStore::CreateCertificateProviderLocked( return nullptr; } CertificateProviderFactory* factory = - CertificateProviderRegistry::LookupCertificateProviderFactory( - plugin_config_it->second.plugin_name); + CoreConfiguration::Get() + .certificate_provider_registry() + .LookupCertificateProviderFactory( + plugin_config_it->second.plugin_name); if (factory == nullptr) { // This should never happen since an entry is only inserted in the // plugin_config_map_ if the corresponding factory was found when parsing diff --git a/grpc/src/core/ext/xds/certificate_provider_store.h b/grpc/src/core/ext/xds/certificate_provider_store.h index 9618625c..24b172ac 100644 --- a/grpc/src/core/ext/xds/certificate_provider_store.h +++ b/grpc/src/core/ext/xds/certificate_provider_store.h @@ -16,19 +16,32 @@ // // -#ifndef GRPC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_STORE_H -#define GRPC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_STORE_H +#ifndef GRPC_SRC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_STORE_H +#define GRPC_SRC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_STORE_H #include <grpc/support/port_platform.h> #include <map> +#include <string> +#include <utility> +#include "absl/base/thread_annotations.h" #include "absl/strings/string_view.h" -#include "src/core/ext/xds/certificate_provider_factory.h" +#include <grpc/grpc_security.h> + +#include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/sync.h" +#include "src/core/lib/gprpp/unique_type_name.h" +#include "src/core/lib/gprpp/validation_errors.h" +#include "src/core/lib/iomgr/iomgr_fwd.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_args.h" +#include "src/core/lib/json/json_object_loader.h" +#include "src/core/lib/security/certificate_provider/certificate_provider_factory.h" +#include "src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h" #include "src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h" namespace grpc_core { @@ -40,6 +53,10 @@ class CertificateProviderStore struct PluginDefinition { std::string plugin_name; RefCountedPtr<CertificateProviderFactory::Config> config; + + static const JsonLoaderInterface* JsonLoader(const JsonArgs&); + void JsonPostLoad(const Json& json, const JsonArgs& args, + ValidationErrors* errors); }; // Maps plugin instance (opaque) name to plugin defition. @@ -90,7 +107,7 @@ class CertificateProviderStore static_cast<const grpc_tls_certificate_provider*>(this), other); } - const char* type() const override; + UniqueTypeName type() const override; absl::string_view key() const { return key_; } @@ -118,4 +135,4 @@ class CertificateProviderStore } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_STORE_H +#endif // GRPC_SRC_CORE_EXT_XDS_CERTIFICATE_PROVIDER_STORE_H diff --git a/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.cc b/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.cc index 939eec24..8053fc6e 100644 --- a/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.cc +++ b/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.cc @@ -20,17 +20,26 @@ #include "src/core/ext/xds/file_watcher_certificate_provider_factory.h" +#include <algorithm> +#include <initializer_list> +#include <map> +#include <memory> +#include <vector> + #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" -#include "src/core/ext/xds/certificate_provider_registry.h" -#include "src/core/lib/json/json_util.h" +#include <grpc/support/log.h> +#include <grpc/support/time.h> + +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h" namespace grpc_core { namespace { -const char* kFileWatcherPlugin = "file_watcher"; +constexpr absl::string_view kFileWatcherPlugin = "file_watcher"; } // namespace @@ -38,7 +47,7 @@ const char* kFileWatcherPlugin = "file_watcher"; // FileWatcherCertificateProviderFactory::Config // -const char* FileWatcherCertificateProviderFactory::Config::name() const { +absl::string_view FileWatcherCertificateProviderFactory::Config::name() const { return kFileWatcherPlugin; } @@ -62,59 +71,46 @@ std::string FileWatcherCertificateProviderFactory::Config::ToString() const { return absl::StrJoin(parts, ""); } -RefCountedPtr<FileWatcherCertificateProviderFactory::Config> -FileWatcherCertificateProviderFactory::Config::Parse(const Json& config_json, - grpc_error_handle* error) { - auto config = MakeRefCounted<FileWatcherCertificateProviderFactory::Config>(); - if (config_json.type() != Json::Type::OBJECT) { - *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "error:config type should be OBJECT."); - return nullptr; - } - std::vector<grpc_error_handle> error_list; - ParseJsonObjectField(config_json.object_value(), "certificate_file", - &config->identity_cert_file_, &error_list, false); - ParseJsonObjectField(config_json.object_value(), "private_key_file", - &config->private_key_file_, &error_list, false); - if (config->identity_cert_file_.empty() != - config->private_key_file_.empty()) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( +const JsonLoaderInterface* +FileWatcherCertificateProviderFactory::Config::JsonLoader(const JsonArgs&) { + static const auto* loader = + JsonObjectLoader<Config>() + .OptionalField("certificate_file", &Config::identity_cert_file_) + .OptionalField("private_key_file", &Config::private_key_file_) + .OptionalField("ca_certificate_file", &Config::root_cert_file_) + .OptionalField("refresh_interval", &Config::refresh_interval_) + .Finish(); + return loader; +} + +void FileWatcherCertificateProviderFactory::Config::JsonPostLoad( + const Json& json, const JsonArgs& /*args*/, ValidationErrors* errors) { + if ((json.object().find("certificate_file") == json.object().end()) != + (json.object().find("private_key_file") == json.object().end())) { + errors->AddError( "fields \"certificate_file\" and \"private_key_file\" must be both set " - "or both unset.")); - } - ParseJsonObjectField(config_json.object_value(), "ca_certificate_file", - &config->root_cert_file_, &error_list, false); - if (config->identity_cert_file_.empty() && config->root_cert_file_.empty()) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "At least one of \"certificate_file\" and \"ca_certificate_file\" must " - "be specified.")); + "or both unset"); } - if (!ParseJsonObjectFieldAsDuration( - config_json.object_value(), "refresh_interval", - &config->refresh_interval_, &error_list, false)) { - config->refresh_interval_ = Duration::Minutes(10); // 10 minutes default + if ((json.object().find("certificate_file") == json.object().end()) && + (json.object().find("ca_certificate_file") == json.object().end())) { + errors->AddError( + "at least one of \"certificate_file\" and \"ca_certificate_file\" must " + "be specified"); } - if (!error_list.empty()) { - *error = GRPC_ERROR_CREATE_FROM_VECTOR( - "Error parsing file watcher certificate provider config", &error_list); - return nullptr; - } - return config; } // // FileWatcherCertificateProviderFactory // -const char* FileWatcherCertificateProviderFactory::name() const { +absl::string_view FileWatcherCertificateProviderFactory::name() const { return kFileWatcherPlugin; } RefCountedPtr<CertificateProviderFactory::Config> FileWatcherCertificateProviderFactory::CreateCertificateProviderConfig( - const Json& config_json, grpc_error_handle* error) { - return FileWatcherCertificateProviderFactory::Config::Parse(config_json, - error); + const Json& config_json, const JsonArgs& args, ValidationErrors* errors) { + return LoadFromJson<RefCountedPtr<Config>>(config_json, args, errors); } RefCountedPtr<grpc_tls_certificate_provider> @@ -122,7 +118,7 @@ FileWatcherCertificateProviderFactory::CreateCertificateProvider( RefCountedPtr<CertificateProviderFactory::Config> config) { if (config->name() != name()) { gpr_log(GPR_ERROR, "Wrong config type Actual:%s vs Expected:%s", - config->name(), name()); + std::string(config->name()).c_str(), std::string(name()).c_str()); return nullptr; } auto* file_watcher_config = @@ -134,11 +130,10 @@ FileWatcherCertificateProviderFactory::CreateCertificateProvider( file_watcher_config->refresh_interval().millis() / GPR_MS_PER_SEC); } -void FileWatcherCertificateProviderInit() { - CertificateProviderRegistry::RegisterCertificateProviderFactory( - absl::make_unique<FileWatcherCertificateProviderFactory>()); +void RegisterFileWatcherCertificateProvider( + CoreConfiguration::Builder* builder) { + builder->certificate_provider_registry()->RegisterCertificateProviderFactory( + std::make_unique<FileWatcherCertificateProviderFactory>()); } -void FileWatcherCertificateProviderShutdown() {} - } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.h b/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.h index 10b0037e..7dd552c7 100644 --- a/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.h +++ b/grpc/src/core/ext/xds/file_watcher_certificate_provider_factory.h @@ -16,12 +16,24 @@ // // -#ifndef GRPC_CORE_EXT_XDS_FILE_WATCHER_CERTIFICATE_PROVIDER_FACTORY_H -#define GRPC_CORE_EXT_XDS_FILE_WATCHER_CERTIFICATE_PROVIDER_FACTORY_H +#ifndef GRPC_SRC_CORE_EXT_XDS_FILE_WATCHER_CERTIFICATE_PROVIDER_FACTORY_H +#define GRPC_SRC_CORE_EXT_XDS_FILE_WATCHER_CERTIFICATE_PROVIDER_FACTORY_H #include <grpc/support/port_platform.h> -#include "src/core/ext/xds/certificate_provider_factory.h" +#include <string> + +#include "absl/strings/string_view.h" + +#include <grpc/grpc_security.h> + +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/gprpp/validation_errors.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_args.h" +#include "src/core/lib/json/json_object_loader.h" +#include "src/core/lib/security/certificate_provider/certificate_provider_factory.h" namespace grpc_core { @@ -30,10 +42,7 @@ class FileWatcherCertificateProviderFactory public: class Config : public CertificateProviderFactory::Config { public: - static RefCountedPtr<Config> Parse(const Json& config_json, - grpc_error_handle* error); - - const char* name() const override; + absl::string_view name() const override; std::string ToString() const override; @@ -47,18 +56,22 @@ class FileWatcherCertificateProviderFactory Duration refresh_interval() const { return refresh_interval_; } + static const JsonLoaderInterface* JsonLoader(const JsonArgs& args); + void JsonPostLoad(const Json& json, const JsonArgs& args, + ValidationErrors* errors); + private: std::string identity_cert_file_; std::string private_key_file_; std::string root_cert_file_; - Duration refresh_interval_; + Duration refresh_interval_ = Duration::Minutes(10); }; - const char* name() const override; + absl::string_view name() const override; RefCountedPtr<CertificateProviderFactory::Config> - CreateCertificateProviderConfig(const Json& config_json, - grpc_error_handle* error) override; + CreateCertificateProviderConfig(const Json& config_json, const JsonArgs& args, + ValidationErrors* errors) override; RefCountedPtr<grpc_tls_certificate_provider> CreateCertificateProvider( RefCountedPtr<CertificateProviderFactory::Config> config) override; @@ -66,4 +79,4 @@ class FileWatcherCertificateProviderFactory } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_FILE_WATCHER_CERTIFICATE_PROVIDER_FACTORY_H +#endif // GRPC_SRC_CORE_EXT_XDS_FILE_WATCHER_CERTIFICATE_PROVIDER_FACTORY_H diff --git a/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.cc b/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.cc deleted file mode 100644 index 95137bb3..00000000 --- a/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.cc +++ /dev/null @@ -1,265 +0,0 @@ -// -// -// Copyright 2020 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// - -#include <grpc/support/port_platform.h> - -#include "src/core/ext/xds/google_mesh_ca_certificate_provider_factory.h" - -#include <sstream> -#include <type_traits> - -#include "absl/strings/str_cat.h" - -#include <grpc/support/string_util.h> - -#include "src/core/lib/gpr/string.h" -#include "src/core/lib/iomgr/error.h" -#include "src/core/lib/json/json_util.h" - -namespace grpc_core { - -namespace { - -const char* kMeshCaPlugin = "meshCA"; - -} // namespace - -// -// GoogleMeshCaCertificateProviderFactory::Config -// - -const char* GoogleMeshCaCertificateProviderFactory::Config::name() const { - return kMeshCaPlugin; -} - -std::string GoogleMeshCaCertificateProviderFactory::Config::ToString() const { - // TODO(yashykt): To be filled - return "{}"; -} - -std::vector<grpc_error_handle> -GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectStsService( - const Json::Object& sts_service) { - std::vector<grpc_error_handle> error_list_sts_service; - if (!ParseJsonObjectField(sts_service, "token_exchange_service_uri", - &sts_config_.token_exchange_service_uri, - &error_list_sts_service, false)) { - sts_config_.token_exchange_service_uri = - "securetoken.googleapis.com"; // default - } - ParseJsonObjectField(sts_service, "resource", &sts_config_.resource, - &error_list_sts_service, false); - ParseJsonObjectField(sts_service, "audience", &sts_config_.audience, - &error_list_sts_service, false); - if (!ParseJsonObjectField(sts_service, "scope", &sts_config_.scope, - &error_list_sts_service, false)) { - sts_config_.scope = - "https://www.googleapis.com/auth/cloud-platform"; // default - } - ParseJsonObjectField(sts_service, "requested_token_type", - &sts_config_.requested_token_type, - &error_list_sts_service, false); - ParseJsonObjectField(sts_service, "subject_token_path", - &sts_config_.subject_token_path, - &error_list_sts_service); - ParseJsonObjectField(sts_service, "subject_token_type", - &sts_config_.subject_token_type, - &error_list_sts_service); - ParseJsonObjectField(sts_service, "actor_token_path", - &sts_config_.actor_token_path, &error_list_sts_service, - false); - ParseJsonObjectField(sts_service, "actor_token_type", - &sts_config_.actor_token_type, &error_list_sts_service, - false); - return error_list_sts_service; -} - -std::vector<grpc_error_handle> -GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectCallCredentials( - const Json::Object& call_credentials) { - std::vector<grpc_error_handle> error_list_call_credentials; - const Json::Object* sts_service = nullptr; - if (ParseJsonObjectField(call_credentials, "sts_service", &sts_service, - &error_list_call_credentials)) { - std::vector<grpc_error_handle> error_list_sts_service = - ParseJsonObjectStsService(*sts_service); - if (!error_list_sts_service.empty()) { - error_list_call_credentials.push_back(GRPC_ERROR_CREATE_FROM_VECTOR( - "field:sts_service", &error_list_sts_service)); - } - } - return error_list_call_credentials; -} - -std::vector<grpc_error_handle> -GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectGoogleGrpc( - const Json::Object& google_grpc) { - std::vector<grpc_error_handle> error_list_google_grpc; - if (!ParseJsonObjectField(google_grpc, "target_uri", &endpoint_, - &error_list_google_grpc, false)) { - endpoint_ = "meshca.googleapis.com"; // Default target - } - const Json::Array* call_credentials_array = nullptr; - if (ParseJsonObjectField(google_grpc, "call_credentials", - &call_credentials_array, &error_list_google_grpc)) { - if (call_credentials_array->size() != 1) { - error_list_google_grpc.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:call_credentials error:Need exactly one entry.")); - } else { - const Json::Object* call_credentials = nullptr; - if (ExtractJsonType((*call_credentials_array)[0], "call_credentials[0]", - &call_credentials, &error_list_google_grpc)) { - std::vector<grpc_error_handle> error_list_call_credentials = - ParseJsonObjectCallCredentials(*call_credentials); - if (!error_list_call_credentials.empty()) { - error_list_google_grpc.push_back(GRPC_ERROR_CREATE_FROM_VECTOR( - "field:call_credentials", &error_list_call_credentials)); - } - } - } - } - - return error_list_google_grpc; -} - -std::vector<grpc_error_handle> -GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectGrpcServices( - const Json::Object& grpc_service) { - std::vector<grpc_error_handle> error_list_grpc_services; - const Json::Object* google_grpc = nullptr; - if (ParseJsonObjectField(grpc_service, "google_grpc", &google_grpc, - &error_list_grpc_services)) { - std::vector<grpc_error_handle> error_list_google_grpc = - ParseJsonObjectGoogleGrpc(*google_grpc); - if (!error_list_google_grpc.empty()) { - error_list_grpc_services.push_back(GRPC_ERROR_CREATE_FROM_VECTOR( - "field:google_grpc", &error_list_google_grpc)); - } - } - if (!ParseJsonObjectFieldAsDuration(grpc_service, "timeout", &timeout_, - &error_list_grpc_services, false)) { - timeout_ = Duration::Seconds(10); // 10sec default - } - return error_list_grpc_services; -} - -std::vector<grpc_error_handle> -GoogleMeshCaCertificateProviderFactory::Config::ParseJsonObjectServer( - const Json::Object& server) { - std::vector<grpc_error_handle> error_list_server; - std::string api_type; - if (ParseJsonObjectField(server, "api_type", &api_type, &error_list_server, - false)) { - if (api_type != "GRPC") { - error_list_server.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:api_type error:Only GRPC is supported")); - } - } - const Json::Array* grpc_services = nullptr; - if (ParseJsonObjectField(server, "grpc_services", &grpc_services, - &error_list_server)) { - if (grpc_services->size() != 1) { - error_list_server.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:grpc_services error:Need exactly one entry")); - } else { - const Json::Object* grpc_service = nullptr; - if (ExtractJsonType((*grpc_services)[0], "grpc_services[0]", - &grpc_service, &error_list_server)) { - std::vector<grpc_error_handle> error_list_grpc_services = - ParseJsonObjectGrpcServices(*grpc_service); - if (!error_list_grpc_services.empty()) { - error_list_server.push_back(GRPC_ERROR_CREATE_FROM_VECTOR( - "field:grpc_services", &error_list_grpc_services)); - } - } - } - } - return error_list_server; -} - -RefCountedPtr<GoogleMeshCaCertificateProviderFactory::Config> -GoogleMeshCaCertificateProviderFactory::Config::Parse( - const Json& config_json, grpc_error_handle* error) { - auto config = - MakeRefCounted<GoogleMeshCaCertificateProviderFactory::Config>(); - if (config_json.type() != Json::Type::OBJECT) { - *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "error:config type should be OBJECT."); - return nullptr; - } - std::vector<grpc_error_handle> error_list; - const Json::Object* server = nullptr; - if (ParseJsonObjectField(config_json.object_value(), "server", &server, - &error_list)) { - std::vector<grpc_error_handle> error_list_server = - config->ParseJsonObjectServer(*server); - if (!error_list_server.empty()) { - error_list.push_back( - GRPC_ERROR_CREATE_FROM_VECTOR("field:server", &error_list_server)); - } - } - if (!ParseJsonObjectFieldAsDuration( - config_json.object_value(), "certificate_lifetime", - &config->certificate_lifetime_, &error_list, false)) { - config->certificate_lifetime_ = Duration::Hours(24); // 24hrs default - } - if (!ParseJsonObjectFieldAsDuration( - config_json.object_value(), "renewal_grace_period", - &config->renewal_grace_period_, &error_list, false)) { - config->renewal_grace_period_ = Duration::Hours(12); // 12hrs default - } - std::string key_type; - if (ParseJsonObjectField(config_json.object_value(), "key_type", &key_type, - &error_list, false)) { - if (key_type != "RSA") { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "field:key_type error:Only RSA is supported.")); - } - } - if (!ParseJsonObjectField(config_json.object_value(), "key_size", - &config->key_size_, &error_list, false)) { - config->key_size_ = 2048; // default 2048 bit key size - } - if (!ParseJsonObjectField(config_json.object_value(), "location", - &config->location_, &error_list, false)) { - // GCE/GKE Metadata server needs to be contacted to get the value. - } - if (!error_list.empty()) { - *error = GRPC_ERROR_CREATE_FROM_VECTOR( - "Error parsing google Mesh CA config", &error_list); - return nullptr; - } - return config; -} - -// -// GoogleMeshCaCertificateProviderFactory -// - -const char* GoogleMeshCaCertificateProviderFactory::name() const { - return kMeshCaPlugin; -} - -RefCountedPtr<CertificateProviderFactory::Config> -GoogleMeshCaCertificateProviderFactory::CreateCertificateProviderConfig( - const Json& config_json, grpc_error_handle* error) { - return GoogleMeshCaCertificateProviderFactory::Config::Parse(config_json, - error); -} - -} // namespace grpc_core diff --git a/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.h b/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.h deleted file mode 100644 index 8d1f19a8..00000000 --- a/grpc/src/core/ext/xds/google_mesh_ca_certificate_provider_factory.h +++ /dev/null @@ -1,105 +0,0 @@ -// -// -// Copyright 2020 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// - -#ifndef GRPC_CORE_EXT_XDS_GOOGLE_MESH_CA_CERTIFICATE_PROVIDER_FACTORY_H -#define GRPC_CORE_EXT_XDS_GOOGLE_MESH_CA_CERTIFICATE_PROVIDER_FACTORY_H - -#include <grpc/support/port_platform.h> - -#include "src/core/ext/xds/certificate_provider_factory.h" -#include "src/core/lib/backoff/backoff.h" -#include "src/core/lib/gprpp/ref_counted.h" - -namespace grpc_core { - -class GoogleMeshCaCertificateProviderFactory - : public CertificateProviderFactory { - public: - class Config : public CertificateProviderFactory::Config { - public: - struct StsConfig { - std::string token_exchange_service_uri; - std::string resource; - std::string audience; - std::string scope; - std::string requested_token_type; - std::string subject_token_path; - std::string subject_token_type; - std::string actor_token_path; - std::string actor_token_type; - }; - - const char* name() const override; - - std::string ToString() const override; - - const std::string& endpoint() const { return endpoint_; } - - const StsConfig& sts_config() const { return sts_config_; } - - Duration timeout() const { return timeout_; } - - Duration certificate_lifetime() const { return certificate_lifetime_; } - - Duration renewal_grace_period() const { return renewal_grace_period_; } - - uint32_t key_size() const { return key_size_; } - - const std::string& location() const { return location_; } - - static RefCountedPtr<Config> Parse(const Json& config_json, - grpc_error_handle* error); - - private: - // Helpers for parsing the config - std::vector<grpc_error_handle> ParseJsonObjectStsService( - const Json::Object& sts_service); - std::vector<grpc_error_handle> ParseJsonObjectCallCredentials( - const Json::Object& call_credentials); - std::vector<grpc_error_handle> ParseJsonObjectGoogleGrpc( - const Json::Object& google_grpc); - std::vector<grpc_error_handle> ParseJsonObjectGrpcServices( - const Json::Object& grpc_service); - std::vector<grpc_error_handle> ParseJsonObjectServer( - const Json::Object& server); - - std::string endpoint_; - StsConfig sts_config_; - Duration timeout_; - Duration certificate_lifetime_; - Duration renewal_grace_period_; - uint32_t key_size_; - std::string location_; - }; - - const char* name() const override; - - RefCountedPtr<CertificateProviderFactory::Config> - CreateCertificateProviderConfig(const Json& config_json, - grpc_error_handle* error) override; - - RefCountedPtr<grpc_tls_certificate_provider> CreateCertificateProvider( - RefCountedPtr<CertificateProviderFactory::Config> /*config*/) override { - // TODO(yashykt) : To be implemented - return nullptr; - } -}; - -} // namespace grpc_core - -#endif // GRPC_CORE_EXT_XDS_GOOGLE_MESH_CA_CERTIFICATE_PROVIDER_FACTORY_H diff --git a/grpc/src/core/ext/xds/upb_utils.h b/grpc/src/core/ext/xds/upb_utils.h index be1d7ebb..fd901278 100644 --- a/grpc/src/core/ext/xds/upb_utils.h +++ b/grpc/src/core/ext/xds/upb_utils.h @@ -14,40 +14,18 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_UPB_UTILS_H -#define GRPC_CORE_EXT_XDS_UPB_UTILS_H +#ifndef GRPC_SRC_CORE_EXT_XDS_UPB_UTILS_H +#define GRPC_SRC_CORE_EXT_XDS_UPB_UTILS_H #include <grpc/support/port_platform.h> #include <string> #include "absl/strings/string_view.h" -#include "upb/text_encode.h" -#include "upb/upb.h" -#include "upb/upb.hpp" - -#include "src/core/ext/xds/certificate_provider_store.h" -#include "src/core/ext/xds/xds_bootstrap.h" -#include "src/core/lib/debug/trace.h" +#include "upb/base/string_view.h" namespace grpc_core { -class XdsClient; - -// TODO(roth): Rethink this. All fields except symtab and arena should come -// from XdsClient, injected into XdsResourceType::Decode() somehow without -// passing through XdsApi code, maybe via the AdsResponseParser. -struct XdsEncodingContext { - XdsClient* client; // Used only for logging. Unsafe for dereferencing. - const XdsBootstrap::XdsServer& server; - TraceFlag* tracer; - upb_DefPool* symtab; - upb_Arena* arena; - bool use_v3; - const CertificateProviderStore::PluginDefinitionMap* - certificate_provider_definition_map; -}; - // Works for both std::string and absl::string_view. template <typename T> inline upb_StringView StdStringToUpbString(const T& str) { @@ -64,4 +42,4 @@ inline std::string UpbStringToStdString(const upb_StringView& str) { } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_UPB_UTILS_H +#endif // GRPC_SRC_CORE_EXT_XDS_UPB_UTILS_H diff --git a/grpc/src/core/ext/xds/xds_api.cc b/grpc/src/core/ext/xds/xds_api.cc index 026d53fb..6da587ed 100644 --- a/grpc/src/core/ext/xds/xds_api.cc +++ b/grpc/src/core/ext/xds/xds_api.cc @@ -18,12 +18,16 @@ #include "src/core/ext/xds/xds_api.h" +#include <stdint.h> +#include <stdlib.h> + +#include <algorithm> #include <set> #include <string> #include <vector> #include "absl/strings/str_cat.h" -#include "envoy/admin/v3/config_dump.upb.h" +#include "absl/strings/strip.h" #include "envoy/config/core/v3/base.upb.h" #include "envoy/config/endpoint/v3/load_report.upb.h" #include "envoy/service/discovery/v3/discovery.upb.h" @@ -31,82 +35,51 @@ #include "envoy/service/load_stats/v3/lrs.upb.h" #include "envoy/service/load_stats/v3/lrs.upbdefs.h" #include "envoy/service/status/v3/csds.upb.h" -#include "envoy/service/status/v3/csds.upbdefs.h" #include "google/protobuf/any.upb.h" +#include "google/protobuf/duration.upb.h" #include "google/protobuf/struct.upb.h" #include "google/protobuf/timestamp.upb.h" -#include "google/protobuf/wrappers.upb.h" #include "google/rpc/status.upb.h" -#include "upb/text_encode.h" -#include "upb/upb.h" +#include "upb/base/string_view.h" +#include "upb/reflection/def.h" +#include "upb/text/encode.h" #include "upb/upb.hpp" -#include <grpc/impl/codegen/log.h> -#include <grpc/support/alloc.h> -#include <grpc/support/string_util.h> +#include <grpc/status.h> +#include <grpc/support/log.h> +#include <grpc/support/time.h> #include "src/core/ext/xds/upb_utils.h" -#include "src/core/ext/xds/xds_common_types.h" -#include "src/core/ext/xds/xds_resource_type.h" -#include "src/core/ext/xds/xds_routing.h" -#include "src/core/lib/address_utils/parse_address.h" -#include "src/core/lib/address_utils/sockaddr_utils.h" -#include "src/core/lib/gpr/env.h" -#include "src/core/lib/gpr/string.h" -#include "src/core/lib/gprpp/host_port.h" -#include "src/core/lib/iomgr/error.h" -#include "src/core/lib/iomgr/sockaddr.h" -#include "src/core/lib/iomgr/socket_utils.h" -#include "src/core/lib/slice/slice_internal.h" -#include "src/core/lib/uri/uri_parser.h" +#include "src/core/ext/xds/xds_client.h" +#include "src/core/lib/json/json.h" -namespace grpc_core { +// IWYU pragma: no_include "upb/msg_internal.h" -// If gRPC is built with -DGRPC_XDS_USER_AGENT_NAME_SUFFIX="...", that string -// will be appended to the user agent name reported to the xDS server. -#ifdef GRPC_XDS_USER_AGENT_NAME_SUFFIX -#define GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING \ - " " GRPC_XDS_USER_AGENT_NAME_SUFFIX -#else -#define GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING "" -#endif - -// If gRPC is built with -DGRPC_XDS_USER_AGENT_VERSION_SUFFIX="...", that string -// will be appended to the user agent version reported to the xDS server. -#ifdef GRPC_XDS_USER_AGENT_VERSION_SUFFIX -#define GRPC_XDS_USER_AGENT_VERSION_SUFFIX_STRING \ - " " GRPC_XDS_USER_AGENT_VERSION_SUFFIX -#else -#define GRPC_XDS_USER_AGENT_VERSION_SUFFIX_STRING "" -#endif +namespace grpc_core { XdsApi::XdsApi(XdsClient* client, TraceFlag* tracer, - const XdsBootstrap::Node* node, - const CertificateProviderStore::PluginDefinitionMap* - certificate_provider_definition_map, - upb::SymbolTable* symtab) + const XdsBootstrap::Node* node, upb::SymbolTable* symtab, + std::string user_agent_name, std::string user_agent_version) : client_(client), tracer_(tracer), node_(node), - certificate_provider_definition_map_(certificate_provider_definition_map), symtab_(symtab), - build_version_(absl::StrCat("gRPC C-core ", GPR_PLATFORM_STRING, " ", - grpc_version_string(), - GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING, - GRPC_XDS_USER_AGENT_VERSION_SUFFIX_STRING)), - user_agent_name_(absl::StrCat("gRPC C-core ", GPR_PLATFORM_STRING, - GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING)), - user_agent_version_( - absl::StrCat("C-core ", grpc_version_string(), - GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING, - GRPC_XDS_USER_AGENT_VERSION_SUFFIX_STRING)) {} + user_agent_name_(std::move(user_agent_name)), + user_agent_version_(std::move(user_agent_version)) {} namespace { -void PopulateMetadataValue(const XdsEncodingContext& context, +struct XdsApiContext { + XdsClient* client; + TraceFlag* tracer; + upb_DefPool* symtab; + upb_Arena* arena; +}; + +void PopulateMetadataValue(const XdsApiContext& context, google_protobuf_Value* value_pb, const Json& value); -void PopulateListValue(const XdsEncodingContext& context, +void PopulateListValue(const XdsApiContext& context, google_protobuf_ListValue* list_value, const Json::Array& values) { for (const auto& value : values) { @@ -116,7 +89,7 @@ void PopulateListValue(const XdsEncodingContext& context, } } -void PopulateMetadata(const XdsEncodingContext& context, +void PopulateMetadata(const XdsApiContext& context, google_protobuf_Struct* metadata_pb, const Json::Object& metadata) { for (const auto& p : metadata) { @@ -127,115 +100,74 @@ void PopulateMetadata(const XdsEncodingContext& context, } } -void PopulateMetadataValue(const XdsEncodingContext& context, +void PopulateMetadataValue(const XdsApiContext& context, google_protobuf_Value* value_pb, const Json& value) { switch (value.type()) { - case Json::Type::JSON_NULL: + case Json::Type::kNull: google_protobuf_Value_set_null_value(value_pb, 0); break; - case Json::Type::NUMBER: + case Json::Type::kNumber: google_protobuf_Value_set_number_value( - value_pb, strtod(value.string_value().c_str(), nullptr)); + value_pb, strtod(value.string().c_str(), nullptr)); break; - case Json::Type::STRING: + case Json::Type::kString: google_protobuf_Value_set_string_value( - value_pb, StdStringToUpbString(value.string_value())); + value_pb, StdStringToUpbString(value.string())); break; - case Json::Type::JSON_TRUE: - google_protobuf_Value_set_bool_value(value_pb, true); + case Json::Type::kBoolean: + google_protobuf_Value_set_bool_value(value_pb, value.boolean()); break; - case Json::Type::JSON_FALSE: - google_protobuf_Value_set_bool_value(value_pb, false); - break; - case Json::Type::OBJECT: { + case Json::Type::kObject: { google_protobuf_Struct* struct_value = google_protobuf_Value_mutable_struct_value(value_pb, context.arena); - PopulateMetadata(context, struct_value, value.object_value()); + PopulateMetadata(context, struct_value, value.object()); break; } - case Json::Type::ARRAY: { + case Json::Type::kArray: { google_protobuf_ListValue* list_value = google_protobuf_Value_mutable_list_value(value_pb, context.arena); - PopulateListValue(context, list_value, value.array_value()); + PopulateListValue(context, list_value, value.array()); break; } } } -// Helper functions to manually do protobuf string encoding, so that we -// can populate the node build_version field that was removed in v3. -std::string EncodeVarint(uint64_t val) { - std::string data; - do { - uint8_t byte = val & 0x7fU; - val >>= 7; - if (val) byte |= 0x80U; - data += byte; - } while (val); - return data; -} -std::string EncodeTag(uint32_t field_number, uint8_t wire_type) { - return EncodeVarint((field_number << 3) | wire_type); -} -std::string EncodeStringField(uint32_t field_number, const std::string& str) { - static const uint8_t kDelimitedWireType = 2; - return EncodeTag(field_number, kDelimitedWireType) + - EncodeVarint(str.size()) + str; -} - -void PopulateBuildVersion(const XdsEncodingContext& context, - envoy_config_core_v3_Node* node_msg, - const std::string& build_version) { - std::string encoded_build_version = EncodeStringField(5, build_version); - // TODO(roth): This should use upb_Message_AddUnknown(), but that API is - // broken in the current version of upb, so we're using the internal - // API for now. Change this once we upgrade to a version of upb that - // fixes this bug. - _upb_Message_AddUnknown(node_msg, encoded_build_version.data(), - encoded_build_version.size(), context.arena); -} - -void PopulateNode(const XdsEncodingContext& context, - const XdsBootstrap::Node* node, - const std::string& build_version, +void PopulateNode(const XdsApiContext& context, const XdsBootstrap::Node* node, const std::string& user_agent_name, const std::string& user_agent_version, envoy_config_core_v3_Node* node_msg) { if (node != nullptr) { - if (!node->id.empty()) { + if (!node->id().empty()) { envoy_config_core_v3_Node_set_id(node_msg, - StdStringToUpbString(node->id)); + StdStringToUpbString(node->id())); } - if (!node->cluster.empty()) { + if (!node->cluster().empty()) { envoy_config_core_v3_Node_set_cluster( - node_msg, StdStringToUpbString(node->cluster)); + node_msg, StdStringToUpbString(node->cluster())); } - if (!node->metadata.object_value().empty()) { + if (!node->metadata().empty()) { google_protobuf_Struct* metadata = envoy_config_core_v3_Node_mutable_metadata(node_msg, context.arena); - PopulateMetadata(context, metadata, node->metadata.object_value()); + PopulateMetadata(context, metadata, node->metadata()); } - if (!node->locality_region.empty() || !node->locality_zone.empty() || - !node->locality_sub_zone.empty()) { + if (!node->locality_region().empty() || !node->locality_zone().empty() || + !node->locality_sub_zone().empty()) { envoy_config_core_v3_Locality* locality = envoy_config_core_v3_Node_mutable_locality(node_msg, context.arena); - if (!node->locality_region.empty()) { + if (!node->locality_region().empty()) { envoy_config_core_v3_Locality_set_region( - locality, StdStringToUpbString(node->locality_region)); + locality, StdStringToUpbString(node->locality_region())); } - if (!node->locality_zone.empty()) { + if (!node->locality_zone().empty()) { envoy_config_core_v3_Locality_set_zone( - locality, StdStringToUpbString(node->locality_zone)); + locality, StdStringToUpbString(node->locality_zone())); } - if (!node->locality_sub_zone.empty()) { + if (!node->locality_sub_zone().empty()) { envoy_config_core_v3_Locality_set_sub_zone( - locality, StdStringToUpbString(node->locality_sub_zone)); + locality, StdStringToUpbString(node->locality_sub_zone())); } } } - if (!context.use_v3) { - PopulateBuildVersion(context, node_msg, build_version); - } envoy_config_core_v3_Node_set_user_agent_name( node_msg, StdStringToUpbString(user_agent_name)); envoy_config_core_v3_Node_set_user_agent_version( @@ -247,7 +179,7 @@ void PopulateNode(const XdsEncodingContext& context, } void MaybeLogDiscoveryRequest( - const XdsEncodingContext& context, + const XdsApiContext& context, const envoy_service_discovery_v3_DiscoveryRequest* request) { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { @@ -260,30 +192,23 @@ void MaybeLogDiscoveryRequest( } } -grpc_slice SerializeDiscoveryRequest( - const XdsEncodingContext& context, +std::string SerializeDiscoveryRequest( + const XdsApiContext& context, envoy_service_discovery_v3_DiscoveryRequest* request) { size_t output_length; char* output = envoy_service_discovery_v3_DiscoveryRequest_serialize( request, context.arena, &output_length); - return grpc_slice_from_copied_buffer(output, output_length); + return std::string(output, output_length); } } // namespace -grpc_slice XdsApi::CreateAdsRequest( - const XdsBootstrap::XdsServer& server, absl::string_view type_url, - absl::string_view version, absl::string_view nonce, - const std::vector<std::string>& resource_names, grpc_error_handle error, - bool populate_node) { +std::string XdsApi::CreateAdsRequest( + absl::string_view type_url, absl::string_view version, + absl::string_view nonce, const std::vector<std::string>& resource_names, + absl::Status status, bool populate_node) { upb::Arena arena; - const XdsEncodingContext context = {client_, - server, - tracer_, - symtab_->ptr(), - arena.ptr(), - server.ShouldUseV3(), - certificate_provider_definition_map_}; + const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; // Create a request. envoy_service_discovery_v3_DiscoveryRequest* request = envoy_service_discovery_v3_DiscoveryRequest_new(arena.ptr()); @@ -303,7 +228,7 @@ grpc_slice XdsApi::CreateAdsRequest( } // Set error_detail if it's a NACK. std::string error_string_storage; - if (error != GRPC_ERROR_NONE) { + if (!status.ok()) { google_rpc_Status* error_detail = envoy_service_discovery_v3_DiscoveryRequest_mutable_error_detail( request, arena.ptr()); @@ -312,20 +237,19 @@ grpc_slice XdsApi::CreateAdsRequest( // we could attach a status code to the individual errors where we // generate them in the parsing code, and then use that here. google_rpc_Status_set_code(error_detail, GRPC_STATUS_INVALID_ARGUMENT); - // Error description comes from the error that was passed in. - error_string_storage = grpc_error_std_string(error); + // Error description comes from the status that was passed in. + error_string_storage = std::string(status.message()); upb_StringView error_description = StdStringToUpbString(error_string_storage); google_rpc_Status_set_message(error_detail, error_description); - GRPC_ERROR_UNREF(error); } // Populate node. if (populate_node) { envoy_config_core_v3_Node* node_msg = envoy_service_discovery_v3_DiscoveryRequest_mutable_node(request, arena.ptr()); - PopulateNode(context, node_, build_version_, user_agent_name_, - user_agent_version_, node_msg); + PopulateNode(context, node_, user_agent_name_, user_agent_version_, + node_msg); envoy_config_core_v3_Node_add_client_features( node_msg, upb_StringView_FromString("xds.config.resource-in-sotw"), context.arena); @@ -342,7 +266,7 @@ grpc_slice XdsApi::CreateAdsRequest( namespace { void MaybeLogDiscoveryResponse( - const XdsEncodingContext& context, + const XdsApiContext& context, const envoy_service_discovery_v3_DiscoveryResponse* response) { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { @@ -357,22 +281,14 @@ void MaybeLogDiscoveryResponse( } // namespace -absl::Status XdsApi::ParseAdsResponse(const XdsBootstrap::XdsServer& server, - const grpc_slice& encoded_response, +absl::Status XdsApi::ParseAdsResponse(absl::string_view encoded_response, AdsResponseParserInterface* parser) { upb::Arena arena; - const XdsEncodingContext context = {client_, - server, - tracer_, - symtab_->ptr(), - arena.ptr(), - server.ShouldUseV3(), - certificate_provider_definition_map_}; + const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; // Decode the response. const envoy_service_discovery_v3_DiscoveryResponse* response = envoy_service_discovery_v3_DiscoveryResponse_parse( - reinterpret_cast<const char*>(GRPC_SLICE_START_PTR(encoded_response)), - GRPC_SLICE_LENGTH(encoded_response), arena.ptr()); + encoded_response.data(), encoded_response.size(), arena.ptr()); // If decoding fails, report a fatal error and return. if (response == nullptr) { return absl::InvalidArgumentError("Can't decode DiscoveryResponse."); @@ -403,23 +319,32 @@ absl::Status XdsApi::ParseAdsResponse(const XdsBootstrap::XdsServer& server, absl::string_view serialized_resource = UpbStringToAbsl(google_protobuf_Any_value(resources[i])); // Unwrap Resource messages, if so wrapped. - if (type_url == "envoy.api.v2.Resource" || - type_url == "envoy.service.discovery.v3.Resource") { + absl::string_view resource_name; + if (type_url == "envoy.service.discovery.v3.Resource") { const auto* resource_wrapper = envoy_service_discovery_v3_Resource_parse( serialized_resource.data(), serialized_resource.size(), arena.ptr()); if (resource_wrapper == nullptr) { - return absl::InvalidArgumentError( - "Can't decode Resource proto wrapper"); + parser->ResourceWrapperParsingFailed( + i, "Can't decode Resource proto wrapper"); + continue; } const auto* resource = envoy_service_discovery_v3_Resource_resource(resource_wrapper); + if (resource == nullptr) { + parser->ResourceWrapperParsingFailed( + i, "No resource present in Resource proto wrapper"); + continue; + } type_url = absl::StripPrefix( UpbStringToAbsl(google_protobuf_Any_type_url(resource)), "type.googleapis.com/"); serialized_resource = UpbStringToAbsl(google_protobuf_Any_value(resource)); + resource_name = UpbStringToAbsl( + envoy_service_discovery_v3_Resource_name(resource_wrapper)); } - parser->ParseResource(context, i, type_url, serialized_resource); + parser->ParseResource(context.arena, i, type_url, resource_name, + serialized_resource); } return absl::OkStatus(); } @@ -427,7 +352,7 @@ absl::Status XdsApi::ParseAdsResponse(const XdsBootstrap::XdsServer& server, namespace { void MaybeLogLrsRequest( - const XdsEncodingContext& context, + const XdsApiContext& context, const envoy_service_load_stats_v3_LoadStatsRequest* request) { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { @@ -440,27 +365,20 @@ void MaybeLogLrsRequest( } } -grpc_slice SerializeLrsRequest( - const XdsEncodingContext& context, +std::string SerializeLrsRequest( + const XdsApiContext& context, const envoy_service_load_stats_v3_LoadStatsRequest* request) { size_t output_length; char* output = envoy_service_load_stats_v3_LoadStatsRequest_serialize( request, context.arena, &output_length); - return grpc_slice_from_copied_buffer(output, output_length); + return std::string(output, output_length); } } // namespace -grpc_slice XdsApi::CreateLrsInitialRequest( - const XdsBootstrap::XdsServer& server) { +std::string XdsApi::CreateLrsInitialRequest() { upb::Arena arena; - const XdsEncodingContext context = {client_, - server, - tracer_, - symtab_->ptr(), - arena.ptr(), - server.ShouldUseV3(), - certificate_provider_definition_map_}; + const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; // Create a request. envoy_service_load_stats_v3_LoadStatsRequest* request = envoy_service_load_stats_v3_LoadStatsRequest_new(arena.ptr()); @@ -468,8 +386,7 @@ grpc_slice XdsApi::CreateLrsInitialRequest( envoy_config_core_v3_Node* node_msg = envoy_service_load_stats_v3_LoadStatsRequest_mutable_node(request, arena.ptr()); - PopulateNode(context, node_, build_version_, user_agent_name_, - user_agent_version_, node_msg); + PopulateNode(context, node_, user_agent_name_, user_agent_version_, node_msg); envoy_config_core_v3_Node_add_client_features( node_msg, upb_StringView_FromString("envoy.lrs.supports_send_all_clusters"), @@ -481,7 +398,7 @@ grpc_slice XdsApi::CreateLrsInitialRequest( namespace { void LocalityStatsPopulate( - const XdsEncodingContext& context, + const XdsApiContext& context, envoy_config_endpoint_v3_UpstreamLocalityStats* output, const XdsLocalityName& locality_name, const XdsClusterLocalityStats::Snapshot& snapshot) { @@ -528,19 +445,10 @@ void LocalityStatsPopulate( } // namespace -grpc_slice XdsApi::CreateLrsRequest( +std::string XdsApi::CreateLrsRequest( ClusterLoadReportMap cluster_load_report_map) { upb::Arena arena; - // The xDS server info is not actually needed here, so we seed it with an - // empty value. - XdsBootstrap::XdsServer empty_server; - const XdsEncodingContext context = {client_, - empty_server, - tracer_, - symtab_->ptr(), - arena.ptr(), - false, - certificate_provider_definition_map_}; + const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; // Create a request. envoy_service_load_stats_v3_LoadStatsRequest* request = envoy_service_load_stats_v3_LoadStatsRequest_new(arena.ptr()); @@ -599,20 +507,39 @@ grpc_slice XdsApi::CreateLrsRequest( return SerializeLrsRequest(context, request); } -grpc_error_handle XdsApi::ParseLrsResponse(const grpc_slice& encoded_response, - bool* send_all_clusters, - std::set<std::string>* cluster_names, - Duration* load_reporting_interval) { +namespace { + +void MaybeLogLrsResponse( + const XdsApiContext& context, + const envoy_service_load_stats_v3_LoadStatsResponse* response) { + if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && + gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { + const upb_MessageDef* msg_type = + envoy_service_load_stats_v3_LoadStatsResponse_getmsgdef(context.symtab); + char buf[10240]; + upb_TextEncode(response, msg_type, nullptr, 0, buf, sizeof(buf)); + gpr_log(GPR_DEBUG, "[xds_client %p] received LRS response: %s", + context.client, buf); + } +} + +} // namespace + +absl::Status XdsApi::ParseLrsResponse(absl::string_view encoded_response, + bool* send_all_clusters, + std::set<std::string>* cluster_names, + Duration* load_reporting_interval) { upb::Arena arena; // Decode the response. const envoy_service_load_stats_v3_LoadStatsResponse* decoded_response = envoy_service_load_stats_v3_LoadStatsResponse_parse( - reinterpret_cast<const char*>(GRPC_SLICE_START_PTR(encoded_response)), - GRPC_SLICE_LENGTH(encoded_response), arena.ptr()); + encoded_response.data(), encoded_response.size(), arena.ptr()); // Parse the response. if (decoded_response == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Can't decode response."); + return absl::UnavailableError("Can't decode response."); } + const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; + MaybeLogLrsResponse(context, decoded_response); // Check send_all_clusters. if (envoy_service_load_stats_v3_LoadStatsResponse_send_all_clusters( decoded_response)) { @@ -634,12 +561,12 @@ grpc_error_handle XdsApi::ParseLrsResponse(const grpc_slice& encoded_response, *load_reporting_interval = Duration::FromSecondsAndNanoseconds( google_protobuf_Duration_seconds(load_reporting_interval_duration), google_protobuf_Duration_nanos(load_reporting_interval_duration)); - return GRPC_ERROR_NONE; + return absl::OkStatus(); } namespace { -google_protobuf_Timestamp* EncodeTimestamp(const XdsEncodingContext& context, +google_protobuf_Timestamp* EncodeTimestamp(const XdsApiContext& context, Timestamp value) { google_protobuf_Timestamp* timestamp = google_protobuf_Timestamp_new(context.arena); @@ -659,18 +586,8 @@ std::string XdsApi::AssembleClientConfig( // Fill-in the node information auto* node = envoy_service_status_v3_ClientConfig_mutable_node(client_config, arena.ptr()); - // The xDS server info is not actually needed here, so we seed it with an - // empty value. - XdsBootstrap::XdsServer empty_server; - const XdsEncodingContext context = {client_, - empty_server, - tracer_, - symtab_->ptr(), - arena.ptr(), - true, - certificate_provider_definition_map_}; - PopulateNode(context, node_, build_version_, user_agent_name_, - user_agent_version_, node); + const XdsApiContext context = {client_, tracer_, symtab_->ptr(), arena.ptr()}; + PopulateNode(context, node_, user_agent_name_, user_agent_version_, node); // Dump each resource. std::vector<std::string> type_url_storage; for (const auto& p : resource_type_metadata_map) { diff --git a/grpc/src/core/ext/xds/xds_api.h b/grpc/src/core/ext/xds/xds_api.h index 959c2d5a..256998cc 100644 --- a/grpc/src/core/ext/xds/xds_api.h +++ b/grpc/src/core/ext/xds/xds_api.h @@ -14,34 +14,36 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_API_H -#define GRPC_CORE_EXT_XDS_XDS_API_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_API_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_API_H #include <grpc/support/port_platform.h> -#include <stdint.h> +#include <stddef.h> +#include <map> #include <set> +#include <string> +#include <utility> +#include <vector> -#include "envoy/admin/v3/config_dump.upb.h" -#include "upb/def.hpp" +#include "absl/status/status.h" +#include "absl/strings/string_view.h" +#include "envoy/admin/v3/config_dump_shared.upb.h" +#include "upb/mem/arena.h" +#include "upb/reflection/def.hpp" -#include <grpc/slice.h> - -#include "src/core/ext/xds/upb_utils.h" #include "src/core/ext/xds/xds_bootstrap.h" #include "src/core/ext/xds/xds_client_stats.h" -#include "src/core/ext/xds/xds_http_filters.h" -#include "src/core/lib/channel/status_util.h" -#include "src/core/lib/matchers/matchers.h" -#include "src/core/lib/resolver/server_address.h" +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/time.h" namespace grpc_core { class XdsClient; // TODO(roth): When we have time, split this into multiple pieces: -// - a common upb-based parsing framework (combine with XdsEncodingContext) // - ADS request/response handling // - LRS request/response handling // - CSDS response generation @@ -65,9 +67,17 @@ class XdsApi { virtual absl::Status ProcessAdsResponseFields(AdsResponseFields fields) = 0; // Called to parse each individual resource in the ADS response. - virtual void ParseResource(const XdsEncodingContext& context, size_t idx, + // Note that resource_name is non-empty only when the resource was + // wrapped in a Resource wrapper proto. + virtual void ParseResource(upb_Arena* arena, size_t idx, absl::string_view type_url, + absl::string_view resource_name, absl::string_view serialized_resource) = 0; + + // Called when a resource is wrapped in a Resource wrapper proto but + // we fail to parse the Resource wrapper. + virtual void ResourceWrapperParsingFailed(size_t idx, + absl::string_view message) = 0; }; struct ClusterLoadReport { @@ -138,37 +148,33 @@ class XdsApi { ""); XdsApi(XdsClient* client, TraceFlag* tracer, const XdsBootstrap::Node* node, - const CertificateProviderStore::PluginDefinitionMap* map, - upb::SymbolTable* symtab); + upb::SymbolTable* symtab, std::string user_agent_name, + std::string user_agent_version); // Creates an ADS request. - // Takes ownership of \a error. - grpc_slice CreateAdsRequest(const XdsBootstrap::XdsServer& server, - absl::string_view type_url, - absl::string_view version, - absl::string_view nonce, - const std::vector<std::string>& resource_names, - grpc_error_handle error, bool populate_node); + std::string CreateAdsRequest(absl::string_view type_url, + absl::string_view version, + absl::string_view nonce, + const std::vector<std::string>& resource_names, + absl::Status status, bool populate_node); // Returns non-OK when failing to deserialize response message. // Otherwise, all events are reported to the parser. - absl::Status ParseAdsResponse(const XdsBootstrap::XdsServer& server, - const grpc_slice& encoded_response, + absl::Status ParseAdsResponse(absl::string_view encoded_response, AdsResponseParserInterface* parser); // Creates an initial LRS request. - grpc_slice CreateLrsInitialRequest(const XdsBootstrap::XdsServer& server); + std::string CreateLrsInitialRequest(); // Creates an LRS request sending a client-side load report. - grpc_slice CreateLrsRequest(ClusterLoadReportMap cluster_load_report_map); + std::string CreateLrsRequest(ClusterLoadReportMap cluster_load_report_map); - // Parses the LRS response and returns \a - // load_reporting_interval for client-side load reporting. If there is any - // error, the output config is invalid. - grpc_error_handle ParseLrsResponse(const grpc_slice& encoded_response, - bool* send_all_clusters, - std::set<std::string>* cluster_names, - Duration* load_reporting_interval); + // Parses the LRS response and populates send_all_clusters, + // cluster_names, and load_reporting_interval. + absl::Status ParseLrsResponse(absl::string_view encoded_response, + bool* send_all_clusters, + std::set<std::string>* cluster_names, + Duration* load_reporting_interval); // Assemble the client config proto message and return the serialized result. std::string AssembleClientConfig( @@ -178,14 +184,11 @@ class XdsApi { XdsClient* client_; TraceFlag* tracer_; const XdsBootstrap::Node* node_; // Do not own. - const CertificateProviderStore::PluginDefinitionMap* - certificate_provider_definition_map_; // Do not own. - upb::SymbolTable* symtab_; // Do not own. - const std::string build_version_; + upb::SymbolTable* symtab_; // Do not own. const std::string user_agent_name_; const std::string user_agent_version_; }; } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_API_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_API_H diff --git a/grpc/src/core/ext/xds/xds_audit_logger_registry.cc b/grpc/src/core/ext/xds/xds_audit_logger_registry.cc new file mode 100644 index 00000000..168e56a6 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_audit_logger_registry.cc @@ -0,0 +1,122 @@ +// +// Copyright 2023 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include <grpc/support/port_platform.h> + +#include "src/core/ext/xds/xds_audit_logger_registry.h" + +#include <string> +#include <utility> + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "envoy/config/core/v3/extension.upb.h" +#include "envoy/config/rbac/v3/rbac.upb.h" + +#include "src/core/ext/xds/xds_common_types.h" +#include "src/core/lib/gprpp/match.h" +#include "src/core/lib/gprpp/validation_errors.h" +#include "src/core/lib/security/authorization/audit_logging.h" + +namespace grpc_core { + +namespace { + +using experimental::AuditLoggerRegistry; + +class StdoutLoggerConfigFactory : public XdsAuditLoggerRegistry::ConfigFactory { + public: + Json::Object ConvertXdsAuditLoggerConfig( + const XdsResourceType::DecodeContext& /*context*/, + absl::string_view /*configuration*/, + ValidationErrors* /*errors*/) override { + // Stdout logger has no configuration right now. So we don't process the + // config protobuf. + return {}; + } + + absl::string_view type() override { return Type(); } + absl::string_view name() override { return "stdout_logger"; } + + static absl::string_view Type() { + return "envoy.extensions.rbac.audit_loggers.stream.v3.StdoutAuditLog"; + } +}; + +} // namespace + +XdsAuditLoggerRegistry::XdsAuditLoggerRegistry() { + audit_logger_config_factories_.emplace( + StdoutLoggerConfigFactory::Type(), + std::make_unique<StdoutLoggerConfigFactory>()); +} + +Json XdsAuditLoggerRegistry::ConvertXdsAuditLoggerConfig( + const XdsResourceType::DecodeContext& context, + const envoy_config_rbac_v3_RBAC_AuditLoggingOptions_AuditLoggerConfig* + logger_config, + ValidationErrors* errors) const { + const auto* typed_extension_config = + envoy_config_rbac_v3_RBAC_AuditLoggingOptions_AuditLoggerConfig_audit_logger( + logger_config); + ValidationErrors::ScopedField field(errors, ".audit_logger"); + if (typed_extension_config == nullptr) { + errors->AddError("field not present"); + return Json(); // A null Json object. + } + ValidationErrors::ScopedField field2(errors, ".typed_config"); + const auto* typed_config = + envoy_config_core_v3_TypedExtensionConfig_typed_config( + typed_extension_config); + auto extension = ExtractXdsExtension(context, typed_config, errors); + if (!extension.has_value()) return Json(); + absl::string_view name; + Json config; + Match( + extension->value, + // Built-in logger types. + [&](absl::string_view serialized_value) { + auto it = audit_logger_config_factories_.find(extension->type); + if (it == audit_logger_config_factories_.end()) return; + name = it->second->name(); + config = Json::FromObject(it->second->ConvertXdsAuditLoggerConfig( + context, serialized_value, errors)); + }, + // Custom logger types. + [&](Json json) { + if (!AuditLoggerRegistry::FactoryExists(extension->type)) return; + name = extension->type; + config = json; + }); + // Config not found in either case if name is empty. + if (name.empty()) { + if (!envoy_config_rbac_v3_RBAC_AuditLoggingOptions_AuditLoggerConfig_is_optional( + logger_config)) { + errors->AddError("unsupported audit logger type"); + } + return Json(); + } + // Validate the converted config. + auto result = AuditLoggerRegistry::ParseConfig(name, config); + if (!result.ok()) { + errors->AddError(result.status().message()); + return Json(); + } + return Json::FromObject({{std::string(name), std::move(config)}}); +} +} // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_audit_logger_registry.h b/grpc/src/core/ext/xds/xds_audit_logger_registry.h new file mode 100644 index 00000000..c363cd40 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_audit_logger_registry.h @@ -0,0 +1,68 @@ +// +// Copyright 2023 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_AUDIT_LOGGER_REGISTRY_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_AUDIT_LOGGER_REGISTRY_H + +#include <grpc/support/port_platform.h> + +#include <map> +#include <memory> + +#include "absl/strings/string_view.h" +#include "envoy/config/rbac/v3/rbac.upb.h" + +#include "src/core/ext/xds/xds_resource_type.h" +#include "src/core/lib/gprpp/validation_errors.h" +#include "src/core/lib/json/json.h" + +namespace grpc_core { + +// A registry that maintains a set of converters that are able to map xDS +// RBAC audit logger configuration to gRPC's JSON format. +class XdsAuditLoggerRegistry { + public: + class ConfigFactory { + public: + virtual ~ConfigFactory() = default; + virtual Json::Object ConvertXdsAuditLoggerConfig( + const XdsResourceType::DecodeContext& context, + absl::string_view configuration, ValidationErrors* errors) = 0; + // The full proto message name for the logger config. + virtual absl::string_view type() = 0; + // The logger name used for the gRPC registry. + virtual absl::string_view name() = 0; + }; + + XdsAuditLoggerRegistry(); + + Json ConvertXdsAuditLoggerConfig( + const XdsResourceType::DecodeContext& context, + const envoy_config_rbac_v3_RBAC_AuditLoggingOptions_AuditLoggerConfig* + logger_config, + ValidationErrors* errors) const; + + private: + // A map of config factories that goes from the type of the audit logging + // config to the config factory. + std::map<absl::string_view /* Owned by ConfigFactory */, + std::unique_ptr<ConfigFactory>> + audit_logger_config_factories_; +}; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_AUDIT_LOGGER_REGISTRY_H diff --git a/grpc/src/core/ext/xds/xds_bootstrap.cc b/grpc/src/core/ext/xds/xds_bootstrap.cc index b61a58e1..72263a72 100644 --- a/grpc/src/core/ext/xds/xds_bootstrap.cc +++ b/grpc/src/core/ext/xds/xds_bootstrap.cc @@ -18,553 +18,21 @@ #include "src/core/ext/xds/xds_bootstrap.h" -#include <errno.h> -#include <stdlib.h> +#include "absl/types/optional.h" -#include <vector> - -#include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" -#include "absl/strings/str_join.h" -#include "absl/strings/string_view.h" - -#include <grpc/grpc_security.h> - -#include "src/core/ext/xds/certificate_provider_registry.h" -#include "src/core/ext/xds/xds_api.h" -#include "src/core/lib/config/core_configuration.h" -#include "src/core/lib/gpr/env.h" #include "src/core/lib/gpr/string.h" -#include "src/core/lib/iomgr/load_file.h" -#include "src/core/lib/json/json_util.h" -#include "src/core/lib/security/credentials/channel_creds_registry.h" -#include "src/core/lib/security/credentials/credentials.h" -#include "src/core/lib/security/credentials/fake/fake_credentials.h" -#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/gprpp/env.h" namespace grpc_core { -// TODO(donnadionne): check to see if federation is enabled, this will be -// removed once federation is fully integrated and enabled by default. +// TODO(roth,apolcyn): remove this federation env var after the 1.55 +// release. bool XdsFederationEnabled() { - char* value = gpr_getenv("GRPC_EXPERIMENTAL_XDS_FEDERATION"); + auto value = GetEnv("GRPC_EXPERIMENTAL_XDS_FEDERATION"); + if (!value.has_value()) return true; bool parsed_value; - bool parse_succeeded = gpr_parse_bool_value(value, &parsed_value); - gpr_free(value); + bool parse_succeeded = gpr_parse_bool_value(value->c_str(), &parsed_value); return parse_succeeded && parsed_value; } -namespace { - -grpc_error_handle ParseChannelCreds(const Json::Object& json, size_t idx, - XdsBootstrap::XdsServer* server) { - std::vector<grpc_error_handle> error_list; - std::string type; - ParseJsonObjectField(json, "type", &type, &error_list); - const Json::Object* config_ptr = nullptr; - ParseJsonObjectField(json, "config", &config_ptr, &error_list, - /*required=*/false); - // Select the first channel creds type that we support. - if (server->channel_creds_type.empty() && - CoreConfiguration::Get().channel_creds_registry().IsSupported(type)) { - Json config; - if (config_ptr != nullptr) config = *config_ptr; - if (!CoreConfiguration::Get().channel_creds_registry().IsValidConfig( - type, config)) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat( - "invalid config for channel creds type \"", type, "\""))); - } - server->channel_creds_type = std::move(type); - server->channel_creds_config = std::move(config); - } - return GRPC_ERROR_CREATE_FROM_VECTOR_AND_CPP_STRING( - absl::StrCat("errors parsing index ", idx), &error_list); -} - -grpc_error_handle ParseChannelCredsArray(const Json::Array& json, - XdsBootstrap::XdsServer* server) { - std::vector<grpc_error_handle> error_list; - for (size_t i = 0; i < json.size(); ++i) { - const Json& child = json.at(i); - if (child.type() != Json::Type::OBJECT) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("array element ", i, " is not an object"))); - } else { - grpc_error_handle parse_error = - ParseChannelCreds(child.object_value(), i, server); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - } - if (server->channel_creds_type.empty()) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "no known creds type found in \"channel_creds\"")); - } - return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing \"channel_creds\" array", - &error_list); -} - -} // namespace - -// -// XdsBootstrap::XdsServer -// - -XdsBootstrap::XdsServer XdsBootstrap::XdsServer::Parse( - const Json& json, grpc_error_handle* error) { - std::vector<grpc_error_handle> error_list; - XdsServer server; - ParseJsonObjectField(json.object_value(), "server_uri", &server.server_uri, - &error_list); - const Json::Array* creds_array = nullptr; - ParseJsonObjectField(json.object_value(), "channel_creds", &creds_array, - &error_list); - if (creds_array != nullptr) { - grpc_error_handle parse_error = - ParseChannelCredsArray(*creds_array, &server); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - const Json::Array* server_features_array = nullptr; - ParseJsonObjectField(json.object_value(), "server_features", - &server_features_array, &error_list, /*required=*/false); - if (server_features_array != nullptr) { - for (const Json& feature_json : *server_features_array) { - if (feature_json.type() == Json::Type::STRING && - feature_json.string_value() == "xds_v3") { - server.server_features.insert(feature_json.string_value()); - } - } - } - *error = GRPC_ERROR_CREATE_FROM_VECTOR_AND_CPP_STRING( - "errors parsing xds server", &error_list); - return server; -} - -Json::Object XdsBootstrap::XdsServer::ToJson() const { - Json::Object channel_creds_json{{"type", channel_creds_type}}; - if (channel_creds_config.type() != Json::Type::JSON_NULL) { - channel_creds_json["config"] = channel_creds_config; - } - Json::Object json{ - {"server_uri", server_uri}, - {"channel_creds", Json::Array{std::move(channel_creds_json)}}, - }; - if (!server_features.empty()) { - Json::Array server_features_json; - for (auto& feature : server_features) { - server_features_json.emplace_back(feature); - } - json["server_features"] = std::move(server_features_json); - } - return json; -} - -bool XdsBootstrap::XdsServer::ShouldUseV3() const { - return server_features.find("xds_v3") != server_features.end(); -} - -// -// XdsBootstrap -// - -std::unique_ptr<XdsBootstrap> XdsBootstrap::Create( - absl::string_view json_string, grpc_error_handle* error) { - Json json = Json::Parse(json_string, error); - if (*error != GRPC_ERROR_NONE) { - grpc_error_handle error_out = - GRPC_ERROR_CREATE_REFERENCING_FROM_STATIC_STRING( - "Failed to parse bootstrap JSON string", error, 1); - GRPC_ERROR_UNREF(*error); - *error = error_out; - return nullptr; - } - return absl::make_unique<XdsBootstrap>(std::move(json), error); -} - -XdsBootstrap::XdsBootstrap(Json json, grpc_error_handle* error) { - if (json.type() != Json::Type::OBJECT) { - *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "malformed JSON in bootstrap file"); - return; - } - std::vector<grpc_error_handle> error_list; - auto it = json.mutable_object()->find("xds_servers"); - if (it == json.mutable_object()->end()) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"xds_servers\" field not present")); - } else if (it->second.type() != Json::Type::ARRAY) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"xds_servers\" field is not an array")); - } else { - grpc_error_handle parse_error = ParseXdsServerList(&it->second, &servers_); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - it = json.mutable_object()->find("node"); - if (it != json.mutable_object()->end()) { - if (it->second.type() != Json::Type::OBJECT) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"node\" field is not an object")); - } else { - grpc_error_handle parse_error = ParseNode(&it->second); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - } - if (XdsFederationEnabled()) { - it = json.mutable_object()->find("authorities"); - if (it != json.mutable_object()->end()) { - if (it->second.type() != Json::Type::OBJECT) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"authorities\" field is not an object")); - } else { - grpc_error_handle parse_error = ParseAuthorities(&it->second); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - } - it = json.mutable_object()->find( - "client_default_listener_resource_name_template"); - if (it != json.mutable_object()->end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"client_default_listener_resource_name_template\" field is not a " - "string")); - } else { - client_default_listener_resource_name_template_ = - std::move(*it->second.mutable_string_value()); - } - } - } - it = json.mutable_object()->find("server_listener_resource_name_template"); - if (it != json.mutable_object()->end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"server_listener_resource_name_template\" field is not a string")); - } else { - server_listener_resource_name_template_ = - std::move(*it->second.mutable_string_value()); - } - } - it = json.mutable_object()->find("certificate_providers"); - if (it != json.mutable_object()->end()) { - if (it->second.type() != Json::Type::OBJECT) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"certificate_providers\" field is not an object")); - } else { - grpc_error_handle parse_error = ParseCertificateProviders(&it->second); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - } - *error = GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing xds bootstrap file", - &error_list); -} - -const XdsBootstrap::Authority* XdsBootstrap::LookupAuthority( - const std::string& name) const { - auto it = authorities_.find(name); - if (it != authorities_.end()) { - return &it->second; - } - return nullptr; -} - -bool XdsBootstrap::XdsServerExists( - const XdsBootstrap::XdsServer& server) const { - if (server == servers_[0]) return true; - for (auto& authority : authorities_) { - for (auto& xds_server : authority.second.xds_servers) { - if (server == xds_server) return true; - } - } - return false; -} - -grpc_error_handle XdsBootstrap::ParseXdsServerList( - Json* json, absl::InlinedVector<XdsServer, 1>* servers) { - std::vector<grpc_error_handle> error_list; - for (size_t i = 0; i < json->mutable_array()->size(); ++i) { - Json& child = json->mutable_array()->at(i); - if (child.type() != Json::Type::OBJECT) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("array element ", i, " is not an object"))); - } else { - grpc_error_handle parse_error; - servers->emplace_back(XdsServer::Parse(child, &parse_error)); - if (parse_error != GRPC_ERROR_NONE) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("errors parsing index ", i))); - error_list.push_back(parse_error); - } - } - } - return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing \"xds_servers\" array", - &error_list); -} - -grpc_error_handle XdsBootstrap::ParseAuthorities(Json* json) { - std::vector<grpc_error_handle> error_list; - for (auto& p : *(json->mutable_object())) { - if (p.second.type() != Json::Type::OBJECT) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING( - "field:authorities element error: element is not a object")); - continue; - } - grpc_error_handle parse_error = ParseAuthority(&p.second, p.first); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing \"authorities\"", - &error_list); -} - -grpc_error_handle XdsBootstrap::ParseAuthority(Json* json, - const std::string& name) { - std::vector<grpc_error_handle> error_list; - Authority authority; - auto it = - json->mutable_object()->find("client_listener_resource_name_template"); - if (it != json->mutable_object()->end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"client_listener_resource_name_template\" field is not a string")); - } else { - std::string expected_prefix = absl::StrCat("xdstp://", name, "/"); - if (!absl::StartsWith(it->second.string_value(), expected_prefix)) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("\"client_listener_resource_name_template\" field " - "must begin with \"", - expected_prefix, "\""))); - } else { - authority.client_listener_resource_name_template = - std::move(*it->second.mutable_string_value()); - } - } - } - it = json->mutable_object()->find("xds_servers"); - if (it != json->mutable_object()->end()) { - if (it->second.type() != Json::Type::ARRAY) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"xds_servers\" field is not an array")); - } else { - grpc_error_handle parse_error = - ParseXdsServerList(&it->second, &authority.xds_servers); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - } - if (error_list.empty()) { - authorities_[name] = std::move(authority); - } - return GRPC_ERROR_CREATE_FROM_VECTOR_AND_CPP_STRING( - absl::StrCat("errors parsing authority ", name), &error_list); -} - -grpc_error_handle XdsBootstrap::ParseNode(Json* json) { - std::vector<grpc_error_handle> error_list; - node_ = absl::make_unique<Node>(); - auto it = json->mutable_object()->find("id"); - if (it != json->mutable_object()->end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("\"id\" field is not a string")); - } else { - node_->id = std::move(*it->second.mutable_string_value()); - } - } - it = json->mutable_object()->find("cluster"); - if (it != json->mutable_object()->end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"cluster\" field is not a string")); - } else { - node_->cluster = std::move(*it->second.mutable_string_value()); - } - } - it = json->mutable_object()->find("locality"); - if (it != json->mutable_object()->end()) { - if (it->second.type() != Json::Type::OBJECT) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"locality\" field is not an object")); - } else { - grpc_error_handle parse_error = ParseLocality(&it->second); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - } - it = json->mutable_object()->find("metadata"); - if (it != json->mutable_object()->end()) { - if (it->second.type() != Json::Type::OBJECT) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"metadata\" field is not an object")); - } else { - node_->metadata = std::move(it->second); - } - } - return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing \"node\" object", - &error_list); -} - -grpc_error_handle XdsBootstrap::ParseLocality(Json* json) { - std::vector<grpc_error_handle> error_list; - auto it = json->mutable_object()->find("region"); - if (it != json->mutable_object()->end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"region\" field is not a string")); - } else { - node_->locality_region = std::move(*it->second.mutable_string_value()); - } - } - it = json->mutable_object()->find("zone"); - if (it != json->mutable_object()->end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"zone\" field is not a string")); - } else { - node_->locality_zone = std::move(*it->second.mutable_string_value()); - } - } - it = json->mutable_object()->find("sub_zone"); - if (it != json->mutable_object()->end()) { - if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"sub_zone\" field is not a string")); - } else { - node_->locality_sub_zone = std::move(*it->second.mutable_string_value()); - } - } - return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing \"locality\" object", - &error_list); -} - -grpc_error_handle XdsBootstrap::ParseCertificateProviders(Json* json) { - std::vector<grpc_error_handle> error_list; - for (auto& certificate_provider : *(json->mutable_object())) { - if (certificate_provider.second.type() != Json::Type::OBJECT) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat( - "element \"", certificate_provider.first, "\" is not an object"))); - } else { - grpc_error_handle parse_error = ParseCertificateProvider( - certificate_provider.first, &certificate_provider.second); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - } - return GRPC_ERROR_CREATE_FROM_VECTOR( - "errors parsing \"certificate_providers\" object", &error_list); -} - -grpc_error_handle XdsBootstrap::ParseCertificateProvider( - const std::string& instance_name, Json* certificate_provider_json) { - std::vector<grpc_error_handle> error_list; - auto it = certificate_provider_json->mutable_object()->find("plugin_name"); - if (it == certificate_provider_json->mutable_object()->end()) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"plugin_name\" field not present")); - } else if (it->second.type() != Json::Type::STRING) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"plugin_name\" field is not a string")); - } else { - std::string plugin_name = std::move(*(it->second.mutable_string_value())); - CertificateProviderFactory* factory = - CertificateProviderRegistry::LookupCertificateProviderFactory( - plugin_name); - if (factory == nullptr) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("Unrecognized plugin name: ", plugin_name))); - } else { - RefCountedPtr<CertificateProviderFactory::Config> config; - it = certificate_provider_json->mutable_object()->find("config"); - if (it != certificate_provider_json->mutable_object()->end()) { - if (it->second.type() != Json::Type::OBJECT) { - error_list.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "\"config\" field is not an object")); - } else { - grpc_error_handle parse_error = GRPC_ERROR_NONE; - config = factory->CreateCertificateProviderConfig(it->second, - &parse_error); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - } else { - // "config" is an optional field, so create an empty JSON object. - grpc_error_handle parse_error = GRPC_ERROR_NONE; - config = factory->CreateCertificateProviderConfig(Json::Object(), - &parse_error); - if (parse_error != GRPC_ERROR_NONE) error_list.push_back(parse_error); - } - certificate_providers_.insert( - {instance_name, {std::move(plugin_name), std::move(config)}}); - } - } - return GRPC_ERROR_CREATE_FROM_VECTOR_AND_CPP_STRING( - absl::StrCat("errors parsing element \"", instance_name, "\""), - &error_list); -} - -std::string XdsBootstrap::ToString() const { - std::vector<std::string> parts; - if (node_ != nullptr) { - parts.push_back(absl::StrFormat( - "node={\n" - " id=\"%s\",\n" - " cluster=\"%s\",\n" - " locality={\n" - " region=\"%s\",\n" - " zone=\"%s\",\n" - " sub_zone=\"%s\"\n" - " },\n" - " metadata=%s,\n" - "},\n", - node_->id, node_->cluster, node_->locality_region, node_->locality_zone, - node_->locality_sub_zone, node_->metadata.Dump())); - } - parts.push_back( - absl::StrFormat("servers=[\n" - " {\n" - " uri=\"%s\",\n" - " creds_type=%s,\n", - server().server_uri, server().channel_creds_type)); - if (server().channel_creds_config.type() != Json::Type::JSON_NULL) { - parts.push_back(absl::StrFormat(" creds_config=%s,", - server().channel_creds_config.Dump())); - } - if (!server().server_features.empty()) { - parts.push_back(absl::StrCat(" server_features=[", - absl::StrJoin(server().server_features, ", "), - "],\n")); - } - parts.push_back(" }\n],\n"); - if (!client_default_listener_resource_name_template_.empty()) { - parts.push_back(absl::StrFormat( - "client_default_listener_resource_name_template=\"%s\",\n", - client_default_listener_resource_name_template_)); - } - if (!server_listener_resource_name_template_.empty()) { - parts.push_back( - absl::StrFormat("server_listener_resource_name_template=\"%s\",\n", - server_listener_resource_name_template_)); - } - parts.push_back("authorities={\n"); - for (const auto& entry : authorities_) { - parts.push_back(absl::StrFormat(" %s={\n", entry.first)); - parts.push_back( - absl::StrFormat(" client_listener_resource_name_template=\"%s\",\n", - entry.second.client_listener_resource_name_template)); - parts.push_back( - absl::StrFormat(" servers=[\n" - " {\n" - " uri=\"%s\",\n" - " creds_type=%s,\n", - entry.second.xds_servers[0].server_uri, - entry.second.xds_servers[0].channel_creds_type)); - parts.push_back(" },\n"); - } - parts.push_back("}"); - parts.push_back("certificate_providers={\n"); - for (const auto& entry : certificate_providers_) { - parts.push_back( - absl::StrFormat(" %s={\n" - " plugin_name=%s\n" - " config=%s\n" - " },\n", - entry.first, entry.second.plugin_name, - entry.second.config->ToString())); - } - parts.push_back("}"); - return absl::StrJoin(parts, ""); -} - } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_bootstrap.h b/grpc/src/core/ext/xds/xds_bootstrap.h index 36f38d99..23ba5ea8 100644 --- a/grpc/src/core/ext/xds/xds_bootstrap.h +++ b/grpc/src/core/ext/xds/xds_bootstrap.h @@ -14,130 +14,75 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_BOOTSTRAP_H -#define GRPC_CORE_EXT_XDS_XDS_BOOTSTRAP_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_BOOTSTRAP_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_BOOTSTRAP_H #include <grpc/support/port_platform.h> -#include <memory> -#include <set> #include <string> -#include <vector> -#include "absl/container/inlined_vector.h" - -#include <grpc/slice.h> - -#include "src/core/ext/xds/certificate_provider_store.h" -#include "src/core/lib/gprpp/memory.h" -#include "src/core/lib/gprpp/ref_counted_ptr.h" -#include "src/core/lib/iomgr/error.h" #include "src/core/lib/json/json.h" -#include "src/core/lib/security/credentials/credentials.h" namespace grpc_core { bool XdsFederationEnabled(); -class XdsClient; - class XdsBootstrap { public: - struct Node { - std::string id; - std::string cluster; - std::string locality_region; - std::string locality_zone; - std::string locality_sub_zone; - Json metadata; + class Node { + public: + virtual ~Node() = default; + + virtual const std::string& id() const = 0; + virtual const std::string& cluster() const = 0; + virtual const std::string& locality_region() const = 0; + virtual const std::string& locality_zone() const = 0; + virtual const std::string& locality_sub_zone() const = 0; + virtual const Json::Object& metadata() const = 0; }; - struct XdsServer { - std::string server_uri; - std::string channel_creds_type; - Json channel_creds_config; - std::set<std::string> server_features; + class XdsServer { + public: + virtual ~XdsServer() = default; - static XdsServer Parse(const Json& json, grpc_error_handle* error); + virtual const std::string& server_uri() const = 0; + virtual bool IgnoreResourceDeletion() const = 0; - bool operator==(const XdsServer& other) const { - return (server_uri == other.server_uri && - channel_creds_type == other.channel_creds_type && - channel_creds_config == other.channel_creds_config && - server_features == other.server_features); - } + virtual bool Equals(const XdsServer& other) const = 0; - bool operator<(const XdsServer& other) const { - if (server_uri < other.server_uri) return true; - if (channel_creds_type < other.channel_creds_type) return true; - if (channel_creds_config.Dump() < other.channel_creds_config.Dump()) { - return true; - } - if (server_features < other.server_features) return true; - return false; + friend bool operator==(const XdsServer& a, const XdsServer& b) { + return a.Equals(b); } - - Json::Object ToJson() const; - - bool ShouldUseV3() const; }; - struct Authority { - std::string client_listener_resource_name_template; - absl::InlinedVector<XdsServer, 1> xds_servers; - }; + class Authority { + public: + virtual ~Authority() = default; - // Creates bootstrap object from json_string. - // If *error is not GRPC_ERROR_NONE after returning, then there was an - // error parsing the contents. - static std::unique_ptr<XdsBootstrap> Create(absl::string_view json_string, - grpc_error_handle* error); + virtual const XdsServer* server() const = 0; + }; - // Do not instantiate directly -- use Create() above instead. - XdsBootstrap(Json json, grpc_error_handle* error); + virtual ~XdsBootstrap() = default; - std::string ToString() const; + virtual std::string ToString() const = 0; // TODO(roth): We currently support only one server. Fix this when we // add support for fallback for the xds channel. - const XdsServer& server() const { return servers_[0]; } - const Node* node() const { return node_.get(); } - const std::string& client_default_listener_resource_name_template() const { - return client_default_listener_resource_name_template_; - } - const std::string& server_listener_resource_name_template() const { - return server_listener_resource_name_template_; - } - const std::map<std::string, Authority>& authorities() const { - return authorities_; - } - const Authority* LookupAuthority(const std::string& name) const; - const CertificateProviderStore::PluginDefinitionMap& certificate_providers() - const { - return certificate_providers_; - } - // A util method to check that an xds server exists in this bootstrap file. - bool XdsServerExists(const XdsServer& server) const; - - private: - grpc_error_handle ParseXdsServerList( - Json* json, absl::InlinedVector<XdsServer, 1>* servers); - grpc_error_handle ParseAuthorities(Json* json); - grpc_error_handle ParseAuthority(Json* json, const std::string& name); - grpc_error_handle ParseNode(Json* json); - grpc_error_handle ParseLocality(Json* json); - grpc_error_handle ParseCertificateProviders(Json* json); - grpc_error_handle ParseCertificateProvider(const std::string& instance_name, - Json* certificate_provider_json); - - absl::InlinedVector<XdsServer, 1> servers_; - std::unique_ptr<Node> node_; - std::string client_default_listener_resource_name_template_; - std::string server_listener_resource_name_template_; - std::map<std::string, Authority> authorities_; - CertificateProviderStore::PluginDefinitionMap certificate_providers_; + virtual const XdsServer& server() const = 0; + + // Returns the node information, or null if not present in the bootstrap + // config. + virtual const Node* node() const = 0; + + // Returns a pointer to the specified authority, or null if it does + // not exist in this bootstrap config. + virtual const Authority* LookupAuthority(const std::string& name) const = 0; + + // If the server exists in the bootstrap config, returns a pointer to + // the XdsServer instance in the config. Otherwise, returns null. + virtual const XdsServer* FindXdsServer(const XdsServer& server) const = 0; }; } // namespace grpc_core -#endif /* GRPC_CORE_EXT_XDS_XDS_BOOTSTRAP_H */ +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_BOOTSTRAP_H diff --git a/grpc/src/core/ext/xds/xds_bootstrap_grpc.cc b/grpc/src/core/ext/xds/xds_bootstrap_grpc.cc new file mode 100644 index 00000000..52e8b31d --- /dev/null +++ b/grpc/src/core/ext/xds/xds_bootstrap_grpc.cc @@ -0,0 +1,374 @@ +// +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include <grpc/support/port_platform.h> + +#include "src/core/ext/xds/xds_bootstrap_grpc.h" + +#include <stdlib.h> + +#include <algorithm> +#include <initializer_list> +#include <set> +#include <utility> +#include <vector> + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/str_format.h" +#include "absl/strings/str_join.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +#include <grpc/support/json.h> + +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_object_loader.h" +#include "src/core/lib/json/json_reader.h" +#include "src/core/lib/json/json_writer.h" +#include "src/core/lib/security/credentials/channel_creds_registry.h" + +namespace grpc_core { + +// +// GrpcXdsBootstrap::GrpcNode::Locality +// + +const JsonLoaderInterface* GrpcXdsBootstrap::GrpcNode::Locality::JsonLoader( + const JsonArgs&) { + static const auto* loader = + JsonObjectLoader<Locality>() + .OptionalField("region", &Locality::region) + .OptionalField("zone", &Locality::zone) + .OptionalField("sub_zone", &Locality::sub_zone) + .Finish(); + return loader; +} + +// +// GrpcXdsBootstrap::GrpcNode +// + +const JsonLoaderInterface* GrpcXdsBootstrap::GrpcNode::JsonLoader( + const JsonArgs&) { + static const auto* loader = + JsonObjectLoader<GrpcNode>() + .OptionalField("id", &GrpcNode::id_) + .OptionalField("cluster", &GrpcNode::cluster_) + .OptionalField("locality", &GrpcNode::locality_) + .OptionalField("metadata", &GrpcNode::metadata_) + .Finish(); + return loader; +} + +// +// GrpcXdsBootstrap::GrpcXdsServer::ChannelCreds +// + +const JsonLoaderInterface* +GrpcXdsBootstrap::GrpcXdsServer::ChannelCreds::JsonLoader(const JsonArgs&) { + static const auto* loader = + JsonObjectLoader<ChannelCreds>() + .Field("type", &ChannelCreds::type) + .OptionalField("config", &ChannelCreds::config) + .Finish(); + return loader; +} + +// +// GrpcXdsBootstrap::GrpcXdsServer +// + +namespace { + +constexpr absl::string_view kServerFeatureIgnoreResourceDeletion = + "ignore_resource_deletion"; + +} // namespace + +bool GrpcXdsBootstrap::GrpcXdsServer::IgnoreResourceDeletion() const { + return server_features_.find(std::string( + kServerFeatureIgnoreResourceDeletion)) != server_features_.end(); +} + +bool GrpcXdsBootstrap::GrpcXdsServer::Equals(const XdsServer& other) const { + const auto& o = static_cast<const GrpcXdsServer&>(other); + return (server_uri_ == o.server_uri_ && + channel_creds_.type == o.channel_creds_.type && + channel_creds_.config == o.channel_creds_.config && + server_features_ == o.server_features_); +} + +const JsonLoaderInterface* GrpcXdsBootstrap::GrpcXdsServer::JsonLoader( + const JsonArgs&) { + static const auto* loader = + JsonObjectLoader<GrpcXdsServer>() + .Field("server_uri", &GrpcXdsServer::server_uri_) + .Finish(); + return loader; +} + +void GrpcXdsBootstrap::GrpcXdsServer::JsonPostLoad(const Json& json, + const JsonArgs& args, + ValidationErrors* errors) { + // Parse "channel_creds". + auto channel_creds_list = LoadJsonObjectField<std::vector<ChannelCreds>>( + json.object(), args, "channel_creds", errors); + if (channel_creds_list.has_value()) { + ValidationErrors::ScopedField field(errors, ".channel_creds"); + for (size_t i = 0; i < channel_creds_list->size(); ++i) { + ValidationErrors::ScopedField field(errors, absl::StrCat("[", i, "]")); + auto& creds = (*channel_creds_list)[i]; + // Select the first channel creds type that we support. + if (channel_creds_.type.empty() && + CoreConfiguration::Get().channel_creds_registry().IsSupported( + creds.type)) { + if (!CoreConfiguration::Get().channel_creds_registry().IsValidConfig( + creds.type, Json::FromObject(creds.config))) { + errors->AddError(absl::StrCat( + "invalid config for channel creds type \"", creds.type, "\"")); + continue; + } + channel_creds_.type = std::move(creds.type); + channel_creds_.config = std::move(creds.config); + } + } + if (channel_creds_.type.empty()) { + errors->AddError("no known creds type found"); + } + } + // Parse "server_features". + { + ValidationErrors::ScopedField field(errors, ".server_features"); + auto it = json.object().find("server_features"); + if (it != json.object().end()) { + if (it->second.type() != Json::Type::kArray) { + errors->AddError("is not an array"); + } else { + const Json::Array& array = it->second.array(); + for (const Json& feature_json : array) { + if (feature_json.type() == Json::Type::kString && + (feature_json.string() == kServerFeatureIgnoreResourceDeletion)) { + server_features_.insert(feature_json.string()); + } + } + } + } + } +} + +Json GrpcXdsBootstrap::GrpcXdsServer::ToJson() const { + Json::Object channel_creds_json{ + {"type", Json::FromString(channel_creds_.type)}, + }; + if (!channel_creds_.config.empty()) { + channel_creds_json["config"] = Json::FromObject(channel_creds_.config); + } + Json::Object json{ + {"server_uri", Json::FromString(server_uri_)}, + {"channel_creds", + Json::FromArray({Json::FromObject(std::move(channel_creds_json))})}, + }; + if (!server_features_.empty()) { + Json::Array server_features_json; + for (auto& feature : server_features_) { + server_features_json.emplace_back(Json::FromString(feature)); + } + json["server_features"] = Json::FromArray(std::move(server_features_json)); + } + return Json::FromObject(std::move(json)); +} + +// +// GrpcXdsBootstrap::GrpcAuthority +// + +const JsonLoaderInterface* GrpcXdsBootstrap::GrpcAuthority::JsonLoader( + const JsonArgs&) { + static const auto* loader = + JsonObjectLoader<GrpcAuthority>() + .OptionalField( + "client_listener_resource_name_template", + &GrpcAuthority::client_listener_resource_name_template_) + .OptionalField("xds_servers", &GrpcAuthority::servers_) + .Finish(); + return loader; +} + +// +// GrpcXdsBootstrap +// + +absl::StatusOr<std::unique_ptr<GrpcXdsBootstrap>> GrpcXdsBootstrap::Create( + absl::string_view json_string) { + auto json = JsonParse(json_string); + if (!json.ok()) { + return absl::InvalidArgumentError(absl::StrCat( + "Failed to parse bootstrap JSON string: ", json.status().ToString())); + } + // Validate JSON. + class XdsJsonArgs : public JsonArgs { + public: + bool IsEnabled(absl::string_view key) const override { + if (key == "federation") return XdsFederationEnabled(); + return true; + } + }; + auto bootstrap = LoadFromJson<GrpcXdsBootstrap>(*json, XdsJsonArgs()); + if (!bootstrap.ok()) return bootstrap.status(); + return std::make_unique<GrpcXdsBootstrap>(std::move(*bootstrap)); +} + +const JsonLoaderInterface* GrpcXdsBootstrap::JsonLoader(const JsonArgs&) { + static const auto* loader = + JsonObjectLoader<GrpcXdsBootstrap>() + .Field("xds_servers", &GrpcXdsBootstrap::servers_) + .OptionalField("node", &GrpcXdsBootstrap::node_) + .OptionalField("certificate_providers", + &GrpcXdsBootstrap::certificate_providers_) + .OptionalField( + "server_listener_resource_name_template", + &GrpcXdsBootstrap::server_listener_resource_name_template_) + .OptionalField("authorities", &GrpcXdsBootstrap::authorities_, + "federation") + .OptionalField("client_default_listener_resource_name_template", + &GrpcXdsBootstrap:: + client_default_listener_resource_name_template_, + "federation") + .Finish(); + return loader; +} + +void GrpcXdsBootstrap::JsonPostLoad(const Json& /*json*/, + const JsonArgs& /*args*/, + ValidationErrors* errors) { + // Verify that there is at least one server present. + { + ValidationErrors::ScopedField field(errors, ".xds_servers"); + if (servers_.empty() && !errors->FieldHasErrors()) { + errors->AddError("must be non-empty"); + } + } + // Verify that each authority has the right prefix in the + // client_listener_resource_name_template field. + { + ValidationErrors::ScopedField field(errors, ".authorities"); + for (const auto& p : authorities_) { + const std::string& name = p.first; + const GrpcAuthority& authority = + static_cast<const GrpcAuthority&>(p.second); + ValidationErrors::ScopedField field( + errors, absl::StrCat("[\"", name, + "\"].client_listener_resource_name_template")); + std::string expected_prefix = absl::StrCat("xdstp://", name, "/"); + if (!authority.client_listener_resource_name_template().empty() && + !absl::StartsWith(authority.client_listener_resource_name_template(), + expected_prefix)) { + errors->AddError( + absl::StrCat("field must begin with \"", expected_prefix, "\"")); + } + } + } +} + +std::string GrpcXdsBootstrap::ToString() const { + std::vector<std::string> parts; + if (node_.has_value()) { + parts.push_back( + absl::StrFormat("node={\n" + " id=\"%s\",\n" + " cluster=\"%s\",\n" + " locality={\n" + " region=\"%s\",\n" + " zone=\"%s\",\n" + " sub_zone=\"%s\"\n" + " },\n" + " metadata=%s,\n" + "},\n", + node_->id(), node_->cluster(), node_->locality_region(), + node_->locality_zone(), node_->locality_sub_zone(), + JsonDump(Json::FromObject(node_->metadata())))); + } + parts.push_back( + absl::StrFormat("servers=[\n%s\n],\n", JsonDump(servers_[0].ToJson()))); + if (!client_default_listener_resource_name_template_.empty()) { + parts.push_back(absl::StrFormat( + "client_default_listener_resource_name_template=\"%s\",\n", + client_default_listener_resource_name_template_)); + } + if (!server_listener_resource_name_template_.empty()) { + parts.push_back( + absl::StrFormat("server_listener_resource_name_template=\"%s\",\n", + server_listener_resource_name_template_)); + } + parts.push_back("authorities={\n"); + for (const auto& entry : authorities_) { + parts.push_back(absl::StrFormat(" %s={\n", entry.first)); + parts.push_back( + absl::StrFormat(" client_listener_resource_name_template=\"%s\",\n", + entry.second.client_listener_resource_name_template())); + if (entry.second.server() != nullptr) { + parts.push_back(absl::StrFormat( + " servers=[\n%s\n],\n", + JsonDump(static_cast<const GrpcXdsServer*>(entry.second.server()) + ->ToJson()))); + } + parts.push_back(" },\n"); + } + parts.push_back("}\n"); + parts.push_back("certificate_providers={\n"); + for (const auto& entry : certificate_providers_) { + parts.push_back( + absl::StrFormat(" %s={\n" + " plugin_name=%s\n" + " config=%s\n" + " },\n", + entry.first, entry.second.plugin_name, + entry.second.config->ToString())); + } + parts.push_back("}"); + return absl::StrJoin(parts, ""); +} + +const XdsBootstrap::Authority* GrpcXdsBootstrap::LookupAuthority( + const std::string& name) const { + auto it = authorities_.find(name); + if (it != authorities_.end()) { + return &it->second; + } + return nullptr; +} + +const XdsBootstrap::XdsServer* GrpcXdsBootstrap::FindXdsServer( + const XdsBootstrap::XdsServer& server) const { + if (static_cast<const GrpcXdsServer&>(server) == servers_[0]) { + return &servers_[0]; + } + for (auto& p : authorities_) { + const auto* authority_server = + static_cast<const GrpcXdsServer*>(p.second.server()); + if (authority_server != nullptr && *authority_server == server) { + return authority_server; + } + } + return nullptr; +} + +} // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_bootstrap_grpc.h b/grpc/src/core/ext/xds/xds_bootstrap_grpc.h new file mode 100644 index 00000000..b3509393 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_bootstrap_grpc.h @@ -0,0 +1,189 @@ +// +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_BOOTSTRAP_GRPC_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_BOOTSTRAP_GRPC_H + +#include <grpc/support/port_platform.h> + +#include <map> +#include <memory> +#include <set> +#include <string> +#include <vector> + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +#include "src/core/ext/xds/certificate_provider_store.h" +#include "src/core/ext/xds/xds_audit_logger_registry.h" +#include "src/core/ext/xds/xds_bootstrap.h" +#include "src/core/ext/xds/xds_cluster_specifier_plugin.h" +#include "src/core/ext/xds/xds_http_filters.h" +#include "src/core/ext/xds/xds_lb_policy_registry.h" +#include "src/core/lib/gprpp/validation_errors.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_args.h" +#include "src/core/lib/json/json_object_loader.h" + +namespace grpc_core { + +class GrpcXdsBootstrap : public XdsBootstrap { + public: + class GrpcNode : public Node { + public: + const std::string& id() const override { return id_; } + const std::string& cluster() const override { return cluster_; } + const std::string& locality_region() const override { + return locality_.region; + } + const std::string& locality_zone() const override { return locality_.zone; } + const std::string& locality_sub_zone() const override { + return locality_.sub_zone; + } + const Json::Object& metadata() const override { return metadata_; } + + static const JsonLoaderInterface* JsonLoader(const JsonArgs&); + + private: + struct Locality { + std::string region; + std::string zone; + std::string sub_zone; + + static const JsonLoaderInterface* JsonLoader(const JsonArgs&); + }; + + std::string id_; + std::string cluster_; + Locality locality_; + Json::Object metadata_; + }; + + class GrpcXdsServer : public XdsServer { + public: + const std::string& server_uri() const override { return server_uri_; } + + bool IgnoreResourceDeletion() const override; + + bool Equals(const XdsServer& other) const override; + + const std::string& channel_creds_type() const { + return channel_creds_.type; + } + const Json::Object& channel_creds_config() const { + return channel_creds_.config; + } + + static const JsonLoaderInterface* JsonLoader(const JsonArgs&); + void JsonPostLoad(const Json& json, const JsonArgs& args, + ValidationErrors* errors); + + Json ToJson() const; + + private: + struct ChannelCreds { + std::string type; + Json::Object config; + + static const JsonLoaderInterface* JsonLoader(const JsonArgs&); + }; + + std::string server_uri_; + ChannelCreds channel_creds_; + std::set<std::string> server_features_; + }; + + class GrpcAuthority : public Authority { + public: + const XdsServer* server() const override { + return servers_.empty() ? nullptr : &servers_[0]; + } + + const std::string& client_listener_resource_name_template() const { + return client_listener_resource_name_template_; + } + + static const JsonLoaderInterface* JsonLoader(const JsonArgs&); + + private: + std::vector<GrpcXdsServer> servers_; + std::string client_listener_resource_name_template_; + }; + + // Creates bootstrap object from json_string. + static absl::StatusOr<std::unique_ptr<GrpcXdsBootstrap>> Create( + absl::string_view json_string); + + static const JsonLoaderInterface* JsonLoader(const JsonArgs&); + void JsonPostLoad(const Json& json, const JsonArgs& args, + ValidationErrors* errors); + + std::string ToString() const override; + + const XdsServer& server() const override { return servers_[0]; } + const Node* node() const override { + return node_.has_value() ? &*node_ : nullptr; + } + const Authority* LookupAuthority(const std::string& name) const override; + const XdsServer* FindXdsServer(const XdsServer& server) const override; + + const std::string& client_default_listener_resource_name_template() const { + return client_default_listener_resource_name_template_; + } + const std::string& server_listener_resource_name_template() const { + return server_listener_resource_name_template_; + } + const CertificateProviderStore::PluginDefinitionMap& certificate_providers() + const { + return certificate_providers_; + } + const XdsHttpFilterRegistry& http_filter_registry() const { + return http_filter_registry_; + } + const XdsClusterSpecifierPluginRegistry& cluster_specifier_plugin_registry() + const { + return cluster_specifier_plugin_registry_; + } + const XdsLbPolicyRegistry& lb_policy_registry() const { + return lb_policy_registry_; + } + const XdsAuditLoggerRegistry& audit_logger_registry() const { + return audit_logger_registry_; + } + + // Exposed for testing purposes only. + const std::map<std::string, GrpcAuthority>& authorities() const { + return authorities_; + } + + private: + std::vector<GrpcXdsServer> servers_; + absl::optional<GrpcNode> node_; + std::string client_default_listener_resource_name_template_; + std::string server_listener_resource_name_template_; + std::map<std::string, GrpcAuthority> authorities_; + CertificateProviderStore::PluginDefinitionMap certificate_providers_; + XdsHttpFilterRegistry http_filter_registry_; + XdsClusterSpecifierPluginRegistry cluster_specifier_plugin_registry_; + XdsLbPolicyRegistry lb_policy_registry_; + XdsAuditLoggerRegistry audit_logger_registry_; +}; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_BOOTSTRAP_GRPC_H diff --git a/grpc/src/core/ext/xds/xds_certificate_provider.cc b/grpc/src/core/ext/xds/xds_certificate_provider.cc index 903d4cf3..0ca49793 100644 --- a/grpc/src/core/ext/xds/xds_certificate_provider.cc +++ b/grpc/src/core/ext/xds/xds_certificate_provider.cc @@ -20,8 +20,16 @@ #include "src/core/ext/xds/xds_certificate_provider.h" +#include <utility> + #include "absl/functional/bind_front.h" -#include "absl/strings/str_cat.h" +#include "absl/types/optional.h" + +#include <grpc/support/log.h> + +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/iomgr/error.h" +#include "src/core/lib/security/security_connector/ssl_utils.h" namespace grpc_core { @@ -50,12 +58,11 @@ class RootCertificatesWatcher } void OnError(grpc_error_handle root_cert_error, - grpc_error_handle identity_cert_error) override { - if (root_cert_error != GRPC_ERROR_NONE) { + grpc_error_handle /*identity_cert_error*/) override { + if (!root_cert_error.ok()) { parent_->SetErrorForCert(cert_name_, root_cert_error /* pass the ref */, absl::nullopt); } - GRPC_ERROR_UNREF(identity_cert_error); } private: @@ -84,13 +91,12 @@ class IdentityCertificatesWatcher } } - void OnError(grpc_error_handle root_cert_error, + void OnError(grpc_error_handle /*root_cert_error*/, grpc_error_handle identity_cert_error) override { - if (identity_cert_error != GRPC_ERROR_NONE) { + if (!identity_cert_error.ok()) { parent_->SetErrorForCert(cert_name_, absl::nullopt, identity_cert_error /* pass the ref */); } - GRPC_ERROR_UNREF(root_cert_error); } private: @@ -140,7 +146,7 @@ void XdsCertificateProvider::ClusterCertificateState:: root_cert_watcher_ = nullptr; xds_certificate_provider_->distributor_->SetErrorForCert( "", - GRPC_ERROR_CREATE_FROM_STATIC_STRING( + GRPC_ERROR_CREATE( "No certificate provider available for root certificates"), absl::nullopt); } @@ -171,7 +177,7 @@ void XdsCertificateProvider::ClusterCertificateState:: identity_cert_watcher_ = nullptr; xds_certificate_provider_->distributor_->SetErrorForCert( "", absl::nullopt, - GRPC_ERROR_CREATE_FROM_STATIC_STRING( + GRPC_ERROR_CREATE( "No certificate provider available for identity certificates")); } } @@ -182,7 +188,7 @@ void XdsCertificateProvider::ClusterCertificateState:: void XdsCertificateProvider::ClusterCertificateState::UpdateRootCertWatcher( const std::string& cert_name, grpc_tls_certificate_distributor* root_cert_distributor) { - auto watcher = absl::make_unique<RootCertificatesWatcher>( + auto watcher = std::make_unique<RootCertificatesWatcher>( xds_certificate_provider_->distributor_, cert_name); root_cert_watcher_ = watcher.get(); root_cert_distributor->WatchTlsCertificates(std::move(watcher), @@ -192,7 +198,7 @@ void XdsCertificateProvider::ClusterCertificateState::UpdateRootCertWatcher( void XdsCertificateProvider::ClusterCertificateState::UpdateIdentityCertWatcher( const std::string& cert_name, grpc_tls_certificate_distributor* identity_cert_distributor) { - auto watcher = absl::make_unique<IdentityCertificatesWatcher>( + auto watcher = std::make_unique<IdentityCertificatesWatcher>( xds_certificate_provider_->distributor_, cert_name); identity_cert_watcher_ = watcher.get(); identity_cert_distributor->WatchTlsCertificates( @@ -213,7 +219,7 @@ void XdsCertificateProvider::ClusterCertificateState::WatchStatusCallback( if (root_cert_distributor_ == nullptr) { xds_certificate_provider_->distributor_->SetErrorForCert( cert_name, - GRPC_ERROR_CREATE_FROM_STATIC_STRING( + GRPC_ERROR_CREATE( "No certificate provider available for root certificates"), absl::nullopt); } else { @@ -233,7 +239,7 @@ void XdsCertificateProvider::ClusterCertificateState::WatchStatusCallback( if (identity_cert_distributor_ == nullptr) { xds_certificate_provider_->distributor_->SetErrorForCert( cert_name, absl::nullopt, - GRPC_ERROR_CREATE_FROM_STATIC_STRING( + GRPC_ERROR_CREATE( "No certificate provider available for identity certificates")); } else { UpdateIdentityCertWatcher(cert_name, identity_cert_distributor_.get()); @@ -263,7 +269,10 @@ XdsCertificateProvider::~XdsCertificateProvider() { distributor_->SetWatchStatusCallback(nullptr); } -const char* XdsCertificateProvider::type() const { return "Xds"; } +UniqueTypeName XdsCertificateProvider::type() const { + static UniqueTypeName::Factory kFactory("Xds"); + return kFactory.Create(); +} bool XdsCertificateProvider::ProvidesRootCerts(const std::string& cert_name) { MutexLock lock(&mu_); @@ -278,10 +287,10 @@ void XdsCertificateProvider::UpdateRootCertNameAndDistributor( MutexLock lock(&mu_); auto it = certificate_state_map_.find(cert_name); if (it == certificate_state_map_.end()) { - it = certificate_state_map_ - .emplace(cert_name, - absl::make_unique<ClusterCertificateState>(this)) - .first; + it = + certificate_state_map_ + .emplace(cert_name, std::make_unique<ClusterCertificateState>(this)) + .first; } it->second->UpdateRootCertNameAndDistributor(cert_name, root_cert_name, root_cert_distributor); @@ -303,10 +312,10 @@ void XdsCertificateProvider::UpdateIdentityCertNameAndDistributor( MutexLock lock(&mu_); auto it = certificate_state_map_.find(cert_name); if (it == certificate_state_map_.end()) { - it = certificate_state_map_ - .emplace(cert_name, - absl::make_unique<ClusterCertificateState>(this)) - .first; + it = + certificate_state_map_ + .emplace(cert_name, std::make_unique<ClusterCertificateState>(this)) + .first; } it->second->UpdateIdentityCertNameAndDistributor( cert_name, identity_cert_name, identity_cert_distributor); @@ -354,10 +363,10 @@ void XdsCertificateProvider::WatchStatusCallback(std::string cert_name, MutexLock lock(&mu_); auto it = certificate_state_map_.find(cert_name); if (it == certificate_state_map_.end()) { - it = certificate_state_map_ - .emplace(cert_name, - absl::make_unique<ClusterCertificateState>(this)) - .first; + it = + certificate_state_map_ + .emplace(cert_name, std::make_unique<ClusterCertificateState>(this)) + .first; } it->second->WatchStatusCallback(cert_name, root_being_watched, identity_being_watched); diff --git a/grpc/src/core/ext/xds/xds_certificate_provider.h b/grpc/src/core/ext/xds/xds_certificate_provider.h index d0f58548..c17a7fa5 100644 --- a/grpc/src/core/ext/xds/xds_certificate_provider.h +++ b/grpc/src/core/ext/xds/xds_certificate_provider.h @@ -16,13 +16,28 @@ // // -#ifndef GRPC_CORE_EXT_XDS_XDS_CERTIFICATE_PROVIDER_H -#define GRPC_CORE_EXT_XDS_XDS_CERTIFICATE_PROVIDER_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_CERTIFICATE_PROVIDER_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_CERTIFICATE_PROVIDER_H #include <grpc/support/port_platform.h> -#include "src/core/ext/xds/xds_api.h" +#include <map> +#include <memory> +#include <string> +#include <vector> + +#include "absl/base/thread_annotations.h" +#include "absl/strings/string_view.h" + +#include <grpc/grpc.h> +#include <grpc/grpc_security.h> + +#include "src/core/lib/gpr/useful.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/sync.h" +#include "src/core/lib/gprpp/unique_type_name.h" #include "src/core/lib/matchers/matchers.h" +#include "src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h" #include "src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h" #define GRPC_ARG_XDS_CERTIFICATE_PROVIDER \ @@ -35,11 +50,20 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { XdsCertificateProvider(); ~XdsCertificateProvider() override; + static absl::string_view ChannelArgName() { + return GRPC_ARG_XDS_CERTIFICATE_PROVIDER; + } + + static int ChannelArgsCompare(const XdsCertificateProvider* a, + const XdsCertificateProvider* b) { + return QsortCompare(a, b); + } + RefCountedPtr<grpc_tls_certificate_distributor> distributor() const override { return distributor_; } - const char* type() const override; + UniqueTypeName type() const override; bool ProvidesRootCerts(const std::string& cert_name); void UpdateRootCertNameAndDistributor( @@ -156,4 +180,4 @@ class XdsCertificateProvider : public grpc_tls_certificate_provider { } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_CERTIFICATE_PROVIDER_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_CERTIFICATE_PROVIDER_H diff --git a/grpc/src/core/ext/xds/xds_channel_args.h b/grpc/src/core/ext/xds/xds_channel_args.h index ea6c862b..dfecd44c 100644 --- a/grpc/src/core/ext/xds/xds_channel_args.h +++ b/grpc/src/core/ext/xds/xds_channel_args.h @@ -14,8 +14,8 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_CHANNEL_ARGS_H -#define GRPC_CORE_EXT_XDS_XDS_CHANNEL_ARGS_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_CHANNEL_ARGS_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_CHANNEL_ARGS_H // Specifies channel args for the xDS client. // Used only when GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_BOOTSTRAP_CONFIG @@ -29,4 +29,4 @@ #define GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS \ "grpc.xds_resource_does_not_exist_timeout_ms" -#endif /* GRPC_CORE_EXT_XDS_XDS_CHANNEL_ARGS_H */ +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_CHANNEL_ARGS_H diff --git a/grpc/src/core/ext/xds/xds_channel_stack_modifier.cc b/grpc/src/core/ext/xds/xds_channel_stack_modifier.cc index 194350ec..ff4acbdf 100644 --- a/grpc/src/core/ext/xds/xds_channel_stack_modifier.cc +++ b/grpc/src/core/ext/xds/xds_channel_stack_modifier.cc @@ -20,8 +20,16 @@ #include "src/core/ext/xds/xds_channel_stack_modifier.h" +#include <limits.h> +#include <string.h> + +#include <algorithm> + +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/config/core_configuration.h" -#include "src/core/lib/surface/channel_init.h" +#include "src/core/lib/gpr/useful.h" +#include "src/core/lib/surface/channel_stack_type.h" namespace grpc_core { namespace { @@ -53,9 +61,8 @@ bool XdsChannelStackModifier::ModifyChannelStack(ChannelStackBuilder* builder) { // Insert the filters after the census filter if present. auto it = builder->mutable_stack()->begin(); while (it != builder->mutable_stack()->end()) { - const char* filter_name_at_it = it->filter->name; - if (strcmp("census_server", filter_name_at_it) == 0 || - strcmp("opencensus_server", filter_name_at_it) == 0) { + const char* filter_name_at_it = (*it)->name; + if (strcmp("census_server", filter_name_at_it) == 0) { break; } ++it; @@ -71,8 +78,7 @@ bool XdsChannelStackModifier::ModifyChannelStack(ChannelStackBuilder* builder) { ++it; } for (const grpc_channel_filter* filter : filters_) { - it = builder->mutable_stack()->insert( - it, ChannelStackBuilder::StackEntry{filter, nullptr}); + it = builder->mutable_stack()->insert(it, filter); ++it; } return true; @@ -84,6 +90,10 @@ grpc_arg XdsChannelStackModifier::MakeChannelArg() const { const_cast<XdsChannelStackModifier*>(this), &kChannelArgVtable); } +absl::string_view XdsChannelStackModifier::ChannelArgName() { + return kXdsChannelStackModifierChannelArgName; +} + RefCountedPtr<XdsChannelStackModifier> XdsChannelStackModifier::GetFromChannelArgs(const grpc_channel_args& args) { XdsChannelStackModifier* config_selector_provider = @@ -96,9 +106,8 @@ XdsChannelStackModifier::GetFromChannelArgs(const grpc_channel_args& args) { void RegisterXdsChannelStackModifier(CoreConfiguration::Builder* builder) { builder->channel_init()->RegisterStage( GRPC_SERVER_CHANNEL, INT_MAX, [](ChannelStackBuilder* builder) { - RefCountedPtr<XdsChannelStackModifier> channel_stack_modifier = - XdsChannelStackModifier::GetFromChannelArgs( - *builder->channel_args()); + auto channel_stack_modifier = + builder->channel_args().GetObjectRef<XdsChannelStackModifier>(); if (channel_stack_modifier != nullptr) { return channel_stack_modifier->ModifyChannelStack(builder); } diff --git a/grpc/src/core/ext/xds/xds_channel_stack_modifier.h b/grpc/src/core/ext/xds/xds_channel_stack_modifier.h index 0a164c2c..2a27724f 100644 --- a/grpc/src/core/ext/xds/xds_channel_stack_modifier.h +++ b/grpc/src/core/ext/xds/xds_channel_stack_modifier.h @@ -16,16 +16,23 @@ // // -#ifndef GRPC_CORE_EXT_XDS_XDS_CHANNEL_STACK_MODIFIER_H -#define GRPC_CORE_EXT_XDS_XDS_CHANNEL_STACK_MODIFIER_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_CHANNEL_STACK_MODIFIER_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_CHANNEL_STACK_MODIFIER_H #include <grpc/support/port_platform.h> +#include <utility> #include <vector> -#include "src/core/lib/channel/channel_stack.h" +#include "absl/strings/string_view.h" + +#include <grpc/grpc.h> + +#include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/channel/channel_stack_builder.h" +#include "src/core/lib/gpr/useful.h" #include "src/core/lib/gprpp/ref_counted.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" namespace grpc_core { @@ -43,6 +50,11 @@ class XdsChannelStackModifier : public RefCounted<XdsChannelStackModifier> { grpc_arg MakeChannelArg() const; static RefCountedPtr<XdsChannelStackModifier> GetFromChannelArgs( const grpc_channel_args& args); + static absl::string_view ChannelArgName(); + static int ChannelArgsCompare(const XdsChannelStackModifier* a, + const XdsChannelStackModifier* b) { + return QsortCompare(a, b); + } private: std::vector<const grpc_channel_filter*> filters_; @@ -50,4 +62,4 @@ class XdsChannelStackModifier : public RefCounted<XdsChannelStackModifier> { } // namespace grpc_core -#endif /* GRPC_CORE_EXT_XDS_XDS_CHANNEL_STACK_MODIFIER_H */ +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_CHANNEL_STACK_MODIFIER_H diff --git a/grpc/src/core/ext/xds/xds_client.cc b/grpc/src/core/ext/xds/xds_client.cc index a085d46e..c5a53f66 100644 --- a/grpc/src/core/ext/xds/xds_client.cc +++ b/grpc/src/core/ext/xds/xds_client.cc @@ -19,48 +19,32 @@ #include "src/core/ext/xds/xds_client.h" #include <inttypes.h> -#include <limits.h> #include <string.h> -#include "absl/container/inlined_vector.h" -#include "absl/strings/str_format.h" +#include <algorithm> +#include <type_traits> + +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" +#include "absl/strings/str_split.h" #include "absl/strings/string_view.h" +#include "absl/strings/strip.h" +#include "absl/types/optional.h" +#include "upb/mem/arena.h" -#include <grpc/byte_buffer_reader.h> -#include <grpc/grpc.h> -#include <grpc/support/alloc.h> -#include <grpc/support/time.h> +#include <grpc/event_engine/event_engine.h> +#include <grpc/support/log.h> -#include "src/core/ext/filters/client_channel/client_channel.h" #include "src/core/ext/xds/xds_api.h" #include "src/core/ext/xds/xds_bootstrap.h" -#include "src/core/ext/xds/xds_channel_args.h" #include "src/core/ext/xds/xds_client_stats.h" -#include "src/core/ext/xds/xds_cluster.h" -#include "src/core/ext/xds/xds_cluster_specifier_plugin.h" -#include "src/core/ext/xds/xds_endpoint.h" -#include "src/core/ext/xds/xds_http_filters.h" -#include "src/core/ext/xds/xds_listener.h" -#include "src/core/lib/address_utils/sockaddr_utils.h" #include "src/core/lib/backoff/backoff.h" -#include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/channel/channel_stack.h" -#include "src/core/lib/config/core_configuration.h" -#include "src/core/lib/gpr/env.h" -#include "src/core/lib/gpr/string.h" -#include "src/core/lib/gprpp/memory.h" +#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/sync.h" -#include "src/core/lib/iomgr/sockaddr.h" -#include "src/core/lib/iomgr/timer.h" -#include "src/core/lib/security/credentials/channel_creds_registry.h" -#include "src/core/lib/slice/slice_internal.h" -#include "src/core/lib/slice/slice_string_helpers.h" -#include "src/core/lib/surface/call.h" -#include "src/core/lib/surface/channel.h" -#include "src/core/lib/surface/lame_client.h" +#include "src/core/lib/iomgr/exec_ctx.h" #include "src/core/lib/uri/uri_parser.h" #define GRPC_XDS_INITIAL_CONNECT_BACKOFF_SECONDS 1 @@ -71,19 +55,11 @@ namespace grpc_core { +using ::grpc_event_engine::experimental::EventEngine; + TraceFlag grpc_xds_client_trace(false, "xds_client"); TraceFlag grpc_xds_client_refcount_trace(false, "xds_client_refcount"); -namespace { - -Mutex* g_mu = nullptr; - -const grpc_channel_args* g_channel_args ABSL_GUARDED_BY(*g_mu) = nullptr; -XdsClient* g_xds_client ABSL_GUARDED_BY(*g_mu) = nullptr; -char* g_fallback_bootstrap_config ABSL_GUARDED_BY(*g_mu) = nullptr; - -} // namespace - // // Internal class declarations // @@ -96,9 +72,12 @@ class XdsClient::ChannelState::RetryableCall public: explicit RetryableCall(WeakRefCountedPtr<ChannelState> chand); - void Orphan() override; + // Disable thread-safety analysis because this method is called via + // OrphanablePtr<>, but there's no way to pass the lock annotation + // through there. + void Orphan() override ABSL_NO_THREAD_SAFETY_ANALYSIS; - void OnCallFinishedLocked(); + void OnCallFinishedLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); T* calld() const { return calld_.get(); } ChannelState* chand() const { return chand_.get(); } @@ -107,9 +86,9 @@ class XdsClient::ChannelState::RetryableCall private: void StartNewCallLocked(); - void StartRetryTimerLocked(); - static void OnRetryTimer(void* arg, grpc_error_handle error); - void OnRetryTimerLocked(grpc_error_handle error); + void StartRetryTimerLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + + void OnRetryTimer(); // The wrapped xds call that talks to the xds server. It's instantiated // every time we start a new call. It's null during call retry backoff. @@ -119,9 +98,8 @@ class XdsClient::ChannelState::RetryableCall // Retry state. BackOff backoff_; - grpc_timer retry_timer_; - grpc_closure on_retry_timer_; - bool retry_timer_callback_pending_ = false; + absl::optional<EventEngine::TaskHandle> timer_handle_ + ABSL_GUARDED_BY(&XdsClient::mu_); bool shutting_down_ = false; }; @@ -132,7 +110,6 @@ class XdsClient::ChannelState::AdsCallState public: // The ctor and dtor should not be used directly. explicit AdsCallState(RefCountedPtr<RetryableCall<AdsCallState>> parent); - ~AdsCallState() override; void Orphan() override; @@ -170,106 +147,160 @@ class XdsClient::ChannelState::AdsCallState absl::Status ProcessAdsResponseFields(AdsResponseFields fields) override ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); - void ParseResource(const XdsEncodingContext& context, size_t idx, - absl::string_view type_url, + void ParseResource(upb_Arena* arena, size_t idx, absl::string_view type_url, + absl::string_view resource_name, absl::string_view serialized_resource) override ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + void ResourceWrapperParsingFailed(size_t idx, + absl::string_view message) override; + Result TakeResult() { return std::move(result_); } private: XdsClient* xds_client() const { return ads_call_state_->xds_client(); } AdsCallState* ads_call_state_; - const Timestamp update_time_ = ExecCtx::Get()->Now(); + const Timestamp update_time_ = Timestamp::Now(); Result result_; }; class ResourceTimer : public InternallyRefCounted<ResourceTimer> { public: ResourceTimer(const XdsResourceType* type, const XdsResourceName& name) - : type_(type), name_(name) { - GRPC_CLOSURE_INIT(&timer_callback_, OnTimer, this, - grpc_schedule_on_exec_ctx); - } + : type_(type), name_(name) {} - void Orphan() override { + // Disable thread-safety analysis because this method is called via + // OrphanablePtr<>, but there's no way to pass the lock annotation + // through there. + void Orphan() override ABSL_NO_THREAD_SAFETY_ANALYSIS { MaybeCancelTimer(); Unref(DEBUG_LOCATION, "Orphan"); } - void MaybeStartTimer(RefCountedPtr<AdsCallState> ads_calld) { - if (timer_started_) return; - timer_started_ = true; - ads_calld_ = std::move(ads_calld); - Ref(DEBUG_LOCATION, "timer").release(); - timer_pending_ = true; - grpc_timer_init( - &timer_, - ExecCtx::Get()->Now() + ads_calld_->xds_client()->request_timeout_, - &timer_callback_); + void MarkSubscriptionSendStarted() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) { + subscription_sent_ = true; } - void MaybeCancelTimer() { - if (timer_pending_) { - grpc_timer_cancel(&timer_); - timer_pending_ = false; - } + void MaybeMarkSubscriptionSendComplete( + RefCountedPtr<AdsCallState> ads_calld) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) { + if (subscription_sent_) MaybeStartTimer(std::move(ads_calld)); } - private: - static void OnTimer(void* arg, grpc_error_handle error) { - ResourceTimer* self = static_cast<ResourceTimer*>(arg); - { - MutexLock lock(&self->ads_calld_->xds_client()->mu_); - self->OnTimerLocked(GRPC_ERROR_REF(error)); + void MarkSeen() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) { + resource_seen_ = true; + MaybeCancelTimer(); + } + + void MaybeCancelTimer() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) { + if (timer_handle_.has_value() && + ads_calld_->xds_client()->engine()->Cancel(*timer_handle_)) { + timer_handle_.reset(); } - self->ads_calld_->xds_client()->work_serializer_.DrainQueue(); - self->ads_calld_.reset(); - self->Unref(DEBUG_LOCATION, "timer"); } - void OnTimerLocked(grpc_error_handle error) + private: + void MaybeStartTimer(RefCountedPtr<AdsCallState> ads_calld) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) { - if (error == GRPC_ERROR_NONE && timer_pending_) { - timer_pending_ = false; - absl::Status watcher_error = absl::UnavailableError(absl::StrFormat( - "timeout obtaining resource {type=%s name=%s} from xds server", - type_->type_url(), - XdsClient::ConstructFullXdsResourceName( - name_.authority, type_->type_url(), name_.key))); - if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log(GPR_INFO, "[xds_client %p] xds server %s: %s", - ads_calld_->xds_client(), - ads_calld_->chand()->server_.server_uri.c_str(), - watcher_error.ToString().c_str()); - } + // Don't start timer if we've already either seen the resource or + // marked it as non-existing. + // Note: There are edge cases where we can have seen the resource + // before we have sent the initial subscription request, such as + // when we unsubscribe and then resubscribe to a given resource + // and then get a response containing that resource, all while a + // send_message op is in flight. + if (resource_seen_) return; + // Don't start timer if we haven't yet sent the initial subscription + // request for the resource. + if (!subscription_sent_) return; + // Don't start timer if it's already running. + if (timer_handle_.has_value()) return; + // Check if we already have a cached version of this resource + // (i.e., if this is the initial request for the resource after an + // ADS stream restart). If so, we don't start the timer, because + // (a) we already have the resource and (b) the server may + // optimize by not resending the resource that we already have. + auto& authority_state = + ads_calld->xds_client()->authority_state_map_[name_.authority]; + ResourceState& state = authority_state.resource_map[type_][name_.key]; + if (state.resource != nullptr) return; + // Start timer. + ads_calld_ = std::move(ads_calld); + timer_handle_ = ads_calld_->xds_client()->engine()->RunAfter( + ads_calld_->xds_client()->request_timeout_, + [self = Ref(DEBUG_LOCATION, "timer")]() { + ApplicationCallbackExecCtx callback_exec_ctx; + ExecCtx exec_ctx; + self->OnTimer(); + }); + } + + void OnTimer() { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, + "[xds_client %p] xds server %s: timeout obtaining resource " + "{type=%s name=%s} from xds server", + ads_calld_->xds_client(), + ads_calld_->chand()->server_.server_uri().c_str(), + std::string(type_->type_url()).c_str(), + XdsClient::ConstructFullXdsResourceName( + name_.authority, type_->type_url(), name_.key) + .c_str()); + } + { + MutexLock lock(&ads_calld_->xds_client()->mu_); + timer_handle_.reset(); + resource_seen_ = true; auto& authority_state = ads_calld_->xds_client()->authority_state_map_[name_.authority]; ResourceState& state = authority_state.resource_map[type_][name_.key]; state.meta.client_status = XdsApi::ResourceMetadata::DOES_NOT_EXIST; - ads_calld_->xds_client()->NotifyWatchersOnErrorLocked(state.watchers, - watcher_error); + ads_calld_->xds_client()->NotifyWatchersOnResourceDoesNotExist( + state.watchers); } - GRPC_ERROR_UNREF(error); + ads_calld_->xds_client()->work_serializer_.DrainQueue(); + ads_calld_.reset(); } const XdsResourceType* type_; const XdsResourceName name_; RefCountedPtr<AdsCallState> ads_calld_; - bool timer_started_ = false; - bool timer_pending_ = false; - grpc_timer timer_; - grpc_closure timer_callback_; + // True if we have sent the initial subscription request for this + // resource on this ADS stream. + bool subscription_sent_ ABSL_GUARDED_BY(&XdsClient::mu_) = false; + // True if we have either (a) seen the resource in a response on this + // stream or (b) declared the resource to not exist due to the timer + // firing. + bool resource_seen_ ABSL_GUARDED_BY(&XdsClient::mu_) = false; + absl::optional<EventEngine::TaskHandle> timer_handle_ + ABSL_GUARDED_BY(&XdsClient::mu_); }; - struct ResourceTypeState { - ~ResourceTypeState() { GRPC_ERROR_UNREF(error); } + class StreamEventHandler + : public XdsTransportFactory::XdsTransport::StreamingCall::EventHandler { + public: + explicit StreamEventHandler(RefCountedPtr<AdsCallState> ads_calld) + : ads_calld_(std::move(ads_calld)) {} + + void OnRequestSent(bool ok) override { ads_calld_->OnRequestSent(ok); } + void OnRecvMessage(absl::string_view payload) override { + ads_calld_->OnRecvMessage(payload); + } + void OnStatusReceived(absl::Status status) override { + ads_calld_->OnStatusReceived(std::move(status)); + } + + private: + RefCountedPtr<AdsCallState> ads_calld_; + }; - // Nonce and error for this resource type. + struct ResourceTypeState { + // Nonce and status for this resource type. std::string nonce; - grpc_error_handle error = GRPC_ERROR_NONE; + absl::Status status; // Subscribed resources of this type. std::map<std::string /*authority*/, @@ -280,47 +311,27 @@ class XdsClient::ChannelState::AdsCallState void SendMessageLocked(const XdsResourceType* type) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); - static void OnRequestSent(void* arg, grpc_error_handle error); - void OnRequestSentLocked(grpc_error_handle error) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); - static void OnResponseReceived(void* arg, grpc_error_handle error); - bool OnResponseReceivedLocked() - ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); - static void OnStatusReceived(void* arg, grpc_error_handle error); - void OnStatusReceivedLocked(grpc_error_handle error) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + void OnRequestSent(bool ok); + void OnRecvMessage(absl::string_view payload); + void OnStatusReceived(absl::Status status); bool IsCurrentCallOnChannel() const; // Constructs a list of resource names of a given type for an ADS // request. Also starts the timer for each resource if needed. - std::vector<std::string> ResourceNamesForRequest(const XdsResourceType* type); + std::vector<std::string> ResourceNamesForRequest(const XdsResourceType* type) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); // The owning RetryableCall<>. RefCountedPtr<RetryableCall<AdsCallState>> parent_; + OrphanablePtr<XdsTransportFactory::XdsTransport::StreamingCall> call_; + bool sent_initial_message_ = false; bool seen_response_ = false; - // Always non-NULL. - grpc_call* call_; - - // recv_initial_metadata - grpc_metadata_array initial_metadata_recv_; - - // send_message - grpc_byte_buffer* send_message_payload_ = nullptr; - grpc_closure on_request_sent_; - - // recv_message - grpc_byte_buffer* recv_message_payload_ = nullptr; - grpc_closure on_response_received_; - - // recv_trailing_metadata - grpc_metadata_array trailing_metadata_recv_; - grpc_status_code status_code_; - grpc_slice status_details_; - grpc_closure on_status_received_; + const XdsResourceType* send_message_pending_ + ABSL_GUARDED_BY(&XdsClient::mu_) = nullptr; // Resource types for which requests need to be sent. std::set<const XdsResourceType*> buffered_requests_; @@ -335,11 +346,11 @@ class XdsClient::ChannelState::LrsCallState public: // The ctor and dtor should not be used directly. explicit LrsCallState(RefCountedPtr<RetryableCall<LrsCallState>> parent); - ~LrsCallState() override; void Orphan() override; - void MaybeStartReportingLocked(); + void MaybeStartReportingLocked() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); RetryableCall<LrsCallState>* parent() { return parent_.get(); } ChannelState* chand() const { return parent_->chand(); } @@ -347,30 +358,44 @@ class XdsClient::ChannelState::LrsCallState bool seen_response() const { return seen_response_; } private: + class StreamEventHandler + : public XdsTransportFactory::XdsTransport::StreamingCall::EventHandler { + public: + explicit StreamEventHandler(RefCountedPtr<LrsCallState> lrs_calld) + : lrs_calld_(std::move(lrs_calld)) {} + + void OnRequestSent(bool ok) override { lrs_calld_->OnRequestSent(ok); } + void OnRecvMessage(absl::string_view payload) override { + lrs_calld_->OnRecvMessage(payload); + } + void OnStatusReceived(absl::Status status) override { + lrs_calld_->OnStatusReceived(std::move(status)); + } + + private: + RefCountedPtr<LrsCallState> lrs_calld_; + }; + // Reports client-side load stats according to a fixed interval. class Reporter : public InternallyRefCounted<Reporter> { public: Reporter(RefCountedPtr<LrsCallState> parent, Duration report_interval) : parent_(std::move(parent)), report_interval_(report_interval) { - GRPC_CLOSURE_INIT(&on_next_report_timer_, OnNextReportTimer, this, - grpc_schedule_on_exec_ctx); - GRPC_CLOSURE_INIT(&on_report_done_, OnReportDone, this, - grpc_schedule_on_exec_ctx); ScheduleNextReportLocked(); } - void Orphan() override; + // Disable thread-safety analysis because this method is called via + // OrphanablePtr<>, but there's no way to pass the lock annotation + // through there. + void Orphan() override ABSL_NO_THREAD_SAFETY_ANALYSIS; + + void OnReportDoneLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); private: void ScheduleNextReportLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); - static void OnNextReportTimer(void* arg, grpc_error_handle error); - bool OnNextReportTimerLocked(grpc_error_handle error) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + bool OnNextReportTimer(); bool SendReportLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); - static void OnReportDone(void* arg, grpc_error_handle error); - bool OnReportDoneLocked(grpc_error_handle error) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); bool IsCurrentReporterOnCall() const { return this == parent_->reporter_.get(); @@ -383,47 +408,23 @@ class XdsClient::ChannelState::LrsCallState // The load reporting state. const Duration report_interval_; bool last_report_counters_were_zero_ = false; - bool next_report_timer_callback_pending_ = false; - grpc_timer next_report_timer_; - grpc_closure on_next_report_timer_; - grpc_closure on_report_done_; + absl::optional<EventEngine::TaskHandle> timer_handle_ + ABSL_GUARDED_BY(&XdsClient::mu_); }; - static void OnInitialRequestSent(void* arg, grpc_error_handle error); - void OnInitialRequestSentLocked() - ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); - static void OnResponseReceived(void* arg, grpc_error_handle error); - bool OnResponseReceivedLocked() - ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); - static void OnStatusReceived(void* arg, grpc_error_handle error); - void OnStatusReceivedLocked(grpc_error_handle error) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); + void OnRequestSent(bool ok); + void OnRecvMessage(absl::string_view payload); + void OnStatusReceived(absl::Status status); bool IsCurrentCallOnChannel() const; // The owning RetryableCall<>. RefCountedPtr<RetryableCall<LrsCallState>> parent_; - bool seen_response_ = false; - - // Always non-NULL. - grpc_call* call_; - - // recv_initial_metadata - grpc_metadata_array initial_metadata_recv_; - - // send_message - grpc_byte_buffer* send_message_payload_ = nullptr; - grpc_closure on_initial_request_sent_; - // recv_message - grpc_byte_buffer* recv_message_payload_ = nullptr; - grpc_closure on_response_received_; + OrphanablePtr<XdsTransportFactory::XdsTransport::StreamingCall> call_; - // recv_trailing_metadata - grpc_metadata_array trailing_metadata_recv_; - grpc_status_code status_code_; - grpc_slice status_details_; - grpc_closure on_status_received_; + bool seen_response_ = false; + bool send_message_pending_ ABSL_GUARDED_BY(&XdsClient::mu_) = false; // Load reporting state. bool send_all_clusters_ = false; @@ -433,57 +434,9 @@ class XdsClient::ChannelState::LrsCallState }; // -// XdsClient::ChannelState::StateWatcher -// - -class XdsClient::ChannelState::StateWatcher - : public AsyncConnectivityStateWatcherInterface { - public: - explicit StateWatcher(WeakRefCountedPtr<ChannelState> parent) - : parent_(std::move(parent)) {} - - private: - void OnConnectivityStateChange(grpc_connectivity_state new_state, - const absl::Status& status) override { - { - MutexLock lock(&parent_->xds_client_->mu_); - if (!parent_->shutting_down_ && - new_state == GRPC_CHANNEL_TRANSIENT_FAILURE) { - // In TRANSIENT_FAILURE. Notify all watchers of error. - gpr_log(GPR_INFO, - "[xds_client %p] xds channel for server %s in " - "state TRANSIENT_FAILURE: %s", - parent_->xds_client(), parent_->server_.server_uri.c_str(), - status.ToString().c_str()); - parent_->xds_client_->NotifyOnErrorLocked( - absl::UnavailableError(absl::StrCat( - "xds channel in TRANSIENT_FAILURE, connectivity error: ", - status.ToString()))); - } - } - parent_->xds_client()->work_serializer_.DrainQueue(); - } - - WeakRefCountedPtr<ChannelState> parent_; -}; - -// // XdsClient::ChannelState // -namespace { - -grpc_channel* CreateXdsChannel(grpc_channel_args* args, - const XdsBootstrap::XdsServer& server) { - RefCountedPtr<grpc_channel_credentials> channel_creds = - CoreConfiguration::Get().channel_creds_registry().CreateChannelCreds( - server.channel_creds_type, server.channel_creds_config); - return grpc_channel_create(server.server_uri.c_str(), channel_creds.get(), - args); -} - -} // namespace - XdsClient::ChannelState::ChannelState(WeakRefCountedPtr<XdsClient> xds_client, const XdsBootstrap::XdsServer& server) : DualRefCounted<ChannelState>( @@ -493,38 +446,50 @@ XdsClient::ChannelState::ChannelState(WeakRefCountedPtr<XdsClient> xds_client, xds_client_(std::move(xds_client)), server_(server) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log(GPR_INFO, "[xds_client %p] creating channel to %s", - xds_client_.get(), server.server_uri.c_str()); - } - channel_ = CreateXdsChannel(xds_client_->args_, server); - GPR_ASSERT(channel_ != nullptr); - StartConnectivityWatchLocked(); + gpr_log(GPR_INFO, "[xds_client %p] creating channel %p for server %s", + xds_client_.get(), this, server.server_uri().c_str()); + } + absl::Status status; + transport_ = xds_client_->transport_factory_->Create( + server, + [self = WeakRef(DEBUG_LOCATION, "OnConnectivityFailure")]( + absl::Status status) { + self->OnConnectivityFailure(std::move(status)); + }, + &status); + GPR_ASSERT(transport_ != nullptr); + if (!status.ok()) SetChannelStatusLocked(std::move(status)); } XdsClient::ChannelState::~ChannelState() { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] destroying xds channel %p for server %s", - xds_client(), this, server_.server_uri.c_str()); + xds_client(), this, server_.server_uri().c_str()); } - grpc_channel_destroy(channel_); xds_client_.reset(DEBUG_LOCATION, "ChannelState"); } // This method should only ever be called when holding the lock, but we can't // use a ABSL_EXCLUSIVE_LOCKS_REQUIRED annotation, because Orphan() will be -// called from DualRefCounted::Unref, which cannot have a lock annotation for a -// lock in this subclass. +// called from DualRefCounted::Unref, which cannot have a lock annotation for +// a lock in this subclass. void XdsClient::ChannelState::Orphan() ABSL_NO_THREAD_SAFETY_ANALYSIS { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, "[xds_client %p] orphaning xds channel %p for server %s", + xds_client(), this, server_.server_uri().c_str()); + } shutting_down_ = true; - CancelConnectivityWatchLocked(); + transport_.reset(); // At this time, all strong refs are removed, remove from channel map to - // prevent subsequent subscription from trying to use this ChannelState as it - // is shutting down. - xds_client_->xds_server_channel_map_.erase(server_); + // prevent subsequent subscription from trying to use this ChannelState as + // it is shutting down. + xds_client_->xds_server_channel_map_.erase(&server_); ads_calld_.reset(); lrs_calld_.reset(); } +void XdsClient::ChannelState::ResetBackoff() { transport_->ResetBackoff(); } + XdsClient::ChannelState::AdsCallState* XdsClient::ChannelState::ads_calld() const { return ads_calld_->calld(); @@ -535,10 +500,6 @@ XdsClient::ChannelState::LrsCallState* XdsClient::ChannelState::lrs_calld() return lrs_calld_->calld(); } -bool XdsClient::ChannelState::HasActiveAdsCall() const { - return ads_calld_ != nullptr && ads_calld_->calld() != nullptr; -} - void XdsClient::ChannelState::MaybeStartLrsCall() { if (lrs_calld_ != nullptr) return; lrs_calld_.reset(new RetryableCall<LrsCallState>( @@ -546,43 +507,10 @@ void XdsClient::ChannelState::MaybeStartLrsCall() { } void XdsClient::ChannelState::StopLrsCallLocked() { - xds_client_->xds_load_report_server_map_.erase(server_); + xds_client_->xds_load_report_server_map_.erase(&server_); lrs_calld_.reset(); } -namespace { - -bool IsLameChannel(grpc_channel* channel) { - grpc_channel_element* elem = - grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel)); - return elem->filter == &grpc_lame_filter; -} - -} // namespace - -void XdsClient::ChannelState::StartConnectivityWatchLocked() { - if (IsLameChannel(channel_)) { - xds_client()->NotifyOnErrorLocked( - absl::UnavailableError("xds client has a lame channel")); - return; - } - ClientChannel* client_channel = ClientChannel::GetFromChannel(channel_); - GPR_ASSERT(client_channel != nullptr); - watcher_ = new StateWatcher(WeakRef(DEBUG_LOCATION, "ChannelState+watch")); - client_channel->AddConnectivityWatcher( - GRPC_CHANNEL_IDLE, - OrphanablePtr<AsyncConnectivityStateWatcherInterface>(watcher_)); -} - -void XdsClient::ChannelState::CancelConnectivityWatchLocked() { - if (IsLameChannel(channel_)) { - return; - } - ClientChannel* client_channel = ClientChannel::GetFromChannel(channel_); - GPR_ASSERT(client_channel != nullptr); - client_channel->RemoveConnectivityWatcher(watcher_); -} - void XdsClient::ChannelState::SubscribeLocked(const XdsResourceType* type, const XdsResourceName& name) { if (ads_calld_ == nullptr) { @@ -615,6 +543,56 @@ void XdsClient::ChannelState::UnsubscribeLocked(const XdsResourceType* type, } } +void XdsClient::ChannelState::OnConnectivityFailure(absl::Status status) { + { + MutexLock lock(&xds_client_->mu_); + SetChannelStatusLocked(std::move(status)); + } + xds_client_->work_serializer_.DrainQueue(); +} + +void XdsClient::ChannelState::SetChannelStatusLocked(absl::Status status) { + if (shutting_down_) return; + status = absl::Status(status.code(), absl::StrCat("xDS channel for server ", + server_.server_uri(), ": ", + status.message())); + gpr_log(GPR_INFO, "[xds_client %p] %s", xds_client(), + status.ToString().c_str()); + // If the node ID is set, append that to the status message that we send to + // the watchers, so that it will appear in log messages visible to users. + const auto* node = xds_client_->bootstrap_->node(); + if (node != nullptr) { + status = absl::Status( + status.code(), + absl::StrCat(status.message(), + " (node ID:", xds_client_->bootstrap_->node()->id(), ")")); + } + // Save status in channel, so that we can immediately generate an + // error for any new watchers that may be started. + status_ = status; + // Find all watchers for this channel. + std::set<RefCountedPtr<ResourceWatcherInterface>> watchers; + for (const auto& a : xds_client_->authority_state_map_) { // authority + if (a.second.channel_state != this) continue; + for (const auto& t : a.second.resource_map) { // type + for (const auto& r : t.second) { // resource id + for (const auto& w : r.second.watchers) { // watchers + watchers.insert(w.second); + } + } + } + } + // Enqueue notification for the watchers. + xds_client_->work_serializer_.Schedule( + [watchers = std::move(watchers), status = std::move(status)]() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(xds_client_->work_serializer_) { + for (const auto& watcher : watchers) { + watcher->OnError(status); + } + }, + DEBUG_LOCATION); +} + // // XdsClient::ChannelState::RetryableCall<> // @@ -630,9 +608,6 @@ XdsClient::ChannelState::RetryableCall<T>::RetryableCall( .set_jitter(GRPC_XDS_RECONNECT_JITTER) .set_max_backoff(Duration::Seconds( GRPC_XDS_RECONNECT_MAX_BACKOFF_SECONDS))) { - // Closure Initialization - GRPC_CLOSURE_INIT(&on_retry_timer_, OnRetryTimer, this, - grpc_schedule_on_exec_ctx); StartNewCallLocked(); } @@ -640,7 +615,10 @@ template <typename T> void XdsClient::ChannelState::RetryableCall<T>::Orphan() { shutting_down_ = true; calld_.reset(); - if (retry_timer_callback_pending_) grpc_timer_cancel(&retry_timer_); + if (timer_handle_.has_value()) { + chand()->xds_client()->engine()->Cancel(*timer_handle_); + timer_handle_.reset(); + } this->Unref(DEBUG_LOCATION, "RetryableCall+orphaned"); } @@ -656,13 +634,13 @@ void XdsClient::ChannelState::RetryableCall<T>::OnCallFinishedLocked() { template <typename T> void XdsClient::ChannelState::RetryableCall<T>::StartNewCallLocked() { if (shutting_down_) return; - GPR_ASSERT(chand_->channel_ != nullptr); + GPR_ASSERT(chand_->transport_ != nullptr); GPR_ASSERT(calld_ == nullptr); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log( - GPR_INFO, - "[xds_client %p] xds server %s: start new call from retryable call %p", - chand()->xds_client(), chand()->server_.server_uri.c_str(), this); + gpr_log(GPR_INFO, + "[xds_client %p] xds server %s: start new call from retryable " + "call %p", + chand()->xds_client(), chand()->server_.server_uri().c_str(), this); } calld_ = MakeOrphanable<T>( this->Ref(DEBUG_LOCATION, "RetryableCall+start_new_call")); @@ -672,45 +650,39 @@ template <typename T> void XdsClient::ChannelState::RetryableCall<T>::StartRetryTimerLocked() { if (shutting_down_) return; const Timestamp next_attempt_time = backoff_.NextAttemptTime(); + const Duration timeout = + std::max(next_attempt_time - Timestamp::Now(), Duration::Zero()); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - Duration timeout = - std::max(next_attempt_time - ExecCtx::Get()->Now(), Duration::Zero()); gpr_log(GPR_INFO, "[xds_client %p] xds server %s: call attempt failed; " "retry timer will fire in %" PRId64 "ms.", - chand()->xds_client(), chand()->server_.server_uri.c_str(), + chand()->xds_client(), chand()->server_.server_uri().c_str(), timeout.millis()); } - this->Ref(DEBUG_LOCATION, "RetryableCall+retry_timer_start").release(); - grpc_timer_init(&retry_timer_, next_attempt_time, &on_retry_timer_); - retry_timer_callback_pending_ = true; + timer_handle_ = chand()->xds_client()->engine()->RunAfter( + timeout, + [self = this->Ref(DEBUG_LOCATION, "RetryableCall+retry_timer_start")]() { + ApplicationCallbackExecCtx callback_exec_ctx; + ExecCtx exec_ctx; + self->OnRetryTimer(); + }); } template <typename T> -void XdsClient::ChannelState::RetryableCall<T>::OnRetryTimer( - void* arg, grpc_error_handle error) { - RetryableCall* calld = static_cast<RetryableCall*>(arg); - { - MutexLock lock(&calld->chand_->xds_client()->mu_); - calld->OnRetryTimerLocked(GRPC_ERROR_REF(error)); - } - calld->Unref(DEBUG_LOCATION, "RetryableCall+retry_timer_done"); -} - -template <typename T> -void XdsClient::ChannelState::RetryableCall<T>::OnRetryTimerLocked( - grpc_error_handle error) { - retry_timer_callback_pending_ = false; - if (!shutting_down_ && error == GRPC_ERROR_NONE) { +void XdsClient::ChannelState::RetryableCall<T>::OnRetryTimer() { + MutexLock lock(&chand_->xds_client()->mu_); + if (timer_handle_.has_value()) { + timer_handle_.reset(); + if (shutting_down_) return; if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] xds server %s: retry timer fired (retryable " "call: %p)", - chand()->xds_client(), chand()->server_.server_uri.c_str(), this); + chand()->xds_client(), chand()->server_.server_uri().c_str(), + this); } StartNewCallLocked(); } - GRPC_ERROR_UNREF(error); } // @@ -725,7 +697,7 @@ absl::Status XdsClient::ChannelState::AdsCallState::AdsResponseParser:: "[xds_client %p] xds server %s: received ADS response: type_url=%s, " "version=%s, nonce=%s, num_resources=%" PRIuPTR, ads_call_state_->xds_client(), - ads_call_state_->chand()->server_.server_uri.c_str(), + ads_call_state_->chand()->server_.server_uri().c_str(), fields.type_url.c_str(), fields.version.c_str(), fields.nonce.c_str(), fields.num_resources); } @@ -768,48 +740,68 @@ void UpdateResourceMetadataNacked(const std::string& version, } // namespace void XdsClient::ChannelState::AdsCallState::AdsResponseParser::ParseResource( - const XdsEncodingContext& context, size_t idx, absl::string_view type_url, - absl::string_view serialized_resource) { + upb_Arena* arena, size_t idx, absl::string_view type_url, + absl::string_view resource_name, absl::string_view serialized_resource) { + std::string error_prefix = absl::StrCat( + "resource index ", idx, ": ", + resource_name.empty() ? "" : absl::StrCat(resource_name, ": ")); // Check the type_url of the resource. - bool is_v2 = false; - if (!result_.type->IsType(type_url, &is_v2)) { + if (result_.type_url != type_url) { result_.errors.emplace_back( - absl::StrCat("resource index ", idx, ": incorrect resource type ", - type_url, " (should be ", result_.type_url, ")")); + absl::StrCat(error_prefix, "incorrect resource type \"", type_url, + "\" (should be \"", result_.type_url, "\")")); return; } // Parse the resource. - absl::StatusOr<XdsResourceType::DecodeResult> result = - result_.type->Decode(context, serialized_resource, is_v2); - if (!result.ok()) { + XdsResourceType::DecodeContext context = { + xds_client(), ads_call_state_->chand()->server_, &grpc_xds_client_trace, + xds_client()->symtab_.ptr(), arena}; + XdsResourceType::DecodeResult decode_result = + result_.type->Decode(context, serialized_resource); + // If we didn't already have the resource name from the Resource + // wrapper, try to get it from the decoding result. + if (resource_name.empty()) { + if (decode_result.name.has_value()) { + resource_name = *decode_result.name; + error_prefix = + absl::StrCat("resource index ", idx, ": ", resource_name, ": "); + } else { + // We don't have any way of determining the resource name, so + // there's nothing more we can do here. + result_.errors.emplace_back(absl::StrCat( + error_prefix, decode_result.resource.status().ToString())); + return; + } + } + // If decoding failed, make sure we include the error in the NACK. + const absl::Status& decode_status = decode_result.resource.status(); + if (!decode_status.ok()) { result_.errors.emplace_back( - absl::StrCat("resource index ", idx, ": ", result.status().ToString())); - return; + absl::StrCat(error_prefix, decode_status.ToString())); } // Check the resource name. - auto resource_name = - xds_client()->ParseXdsResourceName(result->name, result_.type); - if (!resource_name.ok()) { - result_.errors.emplace_back(absl::StrCat( - "resource index ", idx, ": Cannot parse xDS resource name \"", - result->name, "\"")); + auto parsed_resource_name = + xds_client()->ParseXdsResourceName(resource_name, result_.type); + if (!parsed_resource_name.ok()) { + result_.errors.emplace_back( + absl::StrCat(error_prefix, "Cannot parse xDS resource name")); return; } // Cancel resource-does-not-exist timer, if needed. auto timer_it = ads_call_state_->state_map_.find(result_.type); if (timer_it != ads_call_state_->state_map_.end()) { - auto it = - timer_it->second.subscribed_resources.find(resource_name->authority); + auto it = timer_it->second.subscribed_resources.find( + parsed_resource_name->authority); if (it != timer_it->second.subscribed_resources.end()) { - auto res_it = it->second.find(resource_name->key); + auto res_it = it->second.find(parsed_resource_name->key); if (res_it != it->second.end()) { - res_it->second->MaybeCancelTimer(); + res_it->second->MarkSeen(); } } } // Lookup the authority in the cache. auto authority_it = - xds_client()->authority_state_map_.find(resource_name->authority); + xds_client()->authority_state_map_.find(parsed_resource_name->authority); if (authority_it == xds_client()->authority_state_map_.end()) { return; // Skip resource -- we don't have a subscription for it. } @@ -821,26 +813,35 @@ void XdsClient::ChannelState::AdsCallState::AdsResponseParser::ParseResource( } auto& type_map = type_it->second; // Found type, so look up resource key. - auto it = type_map.find(resource_name->key); + auto it = type_map.find(parsed_resource_name->key); if (it == type_map.end()) { return; // Skip resource -- we don't have a subscription for it. } ResourceState& resource_state = it->second; // If needed, record that we've seen this resource. if (result_.type->AllResourcesRequiredInSotW()) { - result_.resources_seen[resource_name->authority].insert(resource_name->key); + result_.resources_seen[parsed_resource_name->authority].insert( + parsed_resource_name->key); + } + // If we previously ignored the resource's deletion, log that we're + // now re-adding it. + if (resource_state.ignored_deletion) { + gpr_log(GPR_INFO, + "[xds_client %p] xds server %s: server returned new version of " + "resource for which we previously ignored a deletion: type %s " + "name %s", + xds_client(), + ads_call_state_->chand()->server_.server_uri().c_str(), + std::string(type_url).c_str(), std::string(resource_name).c_str()); + resource_state.ignored_deletion = false; } // Update resource state based on whether the resource is valid. - if (!result->resource.ok()) { - result_.errors.emplace_back(absl::StrCat( - "resource index ", idx, ": ", result->name, - ": validation error: ", result->resource.status().ToString())); + if (!decode_status.ok()) { xds_client()->NotifyWatchersOnErrorLocked( resource_state.watchers, - absl::UnavailableError(absl::StrCat( - "invalid resource: ", result->resource.status().ToString()))); - UpdateResourceMetadataNacked(result_.version, - result->resource.status().ToString(), + absl::UnavailableError( + absl::StrCat("invalid resource: ", decode_status.ToString()))); + UpdateResourceMetadataNacked(result_.version, decode_status.ToString(), update_time_, &resource_state.meta); return; } @@ -849,16 +850,17 @@ void XdsClient::ChannelState::AdsCallState::AdsResponseParser::ParseResource( // If it didn't change, ignore it. if (resource_state.resource != nullptr && result_.type->ResourcesEqual(resource_state.resource.get(), - result->resource->get())) { + decode_result.resource->get())) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] %s resource %s identical to current, ignoring.", - xds_client(), result_.type_url.c_str(), result->name.c_str()); + xds_client(), result_.type_url.c_str(), + std::string(resource_name).c_str()); } return; } // Update the resource state. - resource_state.resource = std::move(*result->resource); + resource_state.resource = std::move(*decode_result.resource); resource_state.meta = CreateResourceMetadataAcked( std::string(serialized_resource), result_.version, update_time_); // Notify watchers. @@ -876,6 +878,12 @@ void XdsClient::ChannelState::AdsCallState::AdsResponseParser::ParseResource( DEBUG_LOCATION); } +void XdsClient::ChannelState::AdsCallState::AdsResponseParser:: + ResourceWrapperParsingFailed(size_t idx, absl::string_view message) { + result_.errors.emplace_back( + absl::StrCat("resource index ", idx, ": ", message)); +} + // // XdsClient::ChannelState::AdsCallState // @@ -887,51 +895,27 @@ XdsClient::ChannelState::AdsCallState::AdsCallState( ? "AdsCallState" : nullptr), parent_(std::move(parent)) { - // Init the ADS call. Note that the call will progress every time there's - // activity in xds_client()->interested_parties_, which is comprised of - // the polling entities from client_channel. GPR_ASSERT(xds_client() != nullptr); - // Create a call with the specified method name. + // Init the ADS call. const char* method = - chand()->server_.ShouldUseV3() - ? "/envoy.service.discovery.v3.AggregatedDiscoveryService/" - "StreamAggregatedResources" - : "/envoy.service.discovery.v2.AggregatedDiscoveryService/" - "StreamAggregatedResources"; - call_ = grpc_channel_create_pollset_set_call( - chand()->channel_, nullptr, GRPC_PROPAGATE_DEFAULTS, - xds_client()->interested_parties_, - StaticSlice::FromStaticString(method).c_slice(), nullptr, - Timestamp::InfFuture(), nullptr); + "/envoy.service.discovery.v3.AggregatedDiscoveryService/" + "StreamAggregatedResources"; + call_ = chand()->transport_->CreateStreamingCall( + method, std::make_unique<StreamEventHandler>( + // Passing the initial ref here. This ref will go away when + // the StreamEventHandler is destroyed. + RefCountedPtr<AdsCallState>(this))); GPR_ASSERT(call_ != nullptr); - // Init data associated with the call. - grpc_metadata_array_init(&initial_metadata_recv_); - grpc_metadata_array_init(&trailing_metadata_recv_); // Start the call. if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] xds server %s: starting ADS call " "(calld: %p, call: %p)", - xds_client(), chand()->server_.server_uri.c_str(), this, call_); - } - // Create the ops. - grpc_call_error call_error; - grpc_op ops[3]; - memset(ops, 0, sizeof(ops)); - // Op: send initial metadata. - grpc_op* op = ops; - op->op = GRPC_OP_SEND_INITIAL_METADATA; - op->data.send_initial_metadata.count = 0; - op->flags = GRPC_INITIAL_METADATA_WAIT_FOR_READY | - GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET; - op->reserved = nullptr; - op++; - call_error = grpc_call_start_batch_and_execute( - call_, ops, static_cast<size_t>(op - ops), nullptr); - GPR_ASSERT(GRPC_CALL_OK == call_error); - // Op: send request message. - GRPC_CLOSURE_INIT(&on_request_sent_, OnRequestSent, this, - grpc_schedule_on_exec_ctx); + xds_client(), chand()->server_.server_uri().c_str(), this, + call_.get()); + } + // If this is a reconnect, add any necessary subscriptions from what's + // already in the cache. for (const auto& a : xds_client()->authority_state_map_) { const std::string& authority = a.first; // Skip authorities that are not using this xDS channel. @@ -944,120 +928,45 @@ XdsClient::ChannelState::AdsCallState::AdsCallState( } } } + // Send initial message if we added any subscriptions above. for (const auto& p : state_map_) { SendMessageLocked(p.first); } - // Op: recv initial metadata. - op = ops; - op->op = GRPC_OP_RECV_INITIAL_METADATA; - op->data.recv_initial_metadata.recv_initial_metadata = - &initial_metadata_recv_; - op->flags = 0; - op->reserved = nullptr; - op++; - // Op: recv response. - op->op = GRPC_OP_RECV_MESSAGE; - op->data.recv_message.recv_message = &recv_message_payload_; - op->flags = 0; - op->reserved = nullptr; - op++; - Ref(DEBUG_LOCATION, "ADS+OnResponseReceivedLocked").release(); - GRPC_CLOSURE_INIT(&on_response_received_, OnResponseReceived, this, - grpc_schedule_on_exec_ctx); - call_error = grpc_call_start_batch_and_execute( - call_, ops, static_cast<size_t>(op - ops), &on_response_received_); - GPR_ASSERT(GRPC_CALL_OK == call_error); - // Op: recv server status. - op = ops; - op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; - op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv_; - op->data.recv_status_on_client.status = &status_code_; - op->data.recv_status_on_client.status_details = &status_details_; - op->flags = 0; - op->reserved = nullptr; - op++; - // This callback signals the end of the call, so it relies on the initial - // ref instead of a new ref. When it's invoked, it's the initial ref that is - // unreffed. - GRPC_CLOSURE_INIT(&on_status_received_, OnStatusReceived, this, - grpc_schedule_on_exec_ctx); - call_error = grpc_call_start_batch_and_execute( - call_, ops, static_cast<size_t>(op - ops), &on_status_received_); - GPR_ASSERT(GRPC_CALL_OK == call_error); -} - -XdsClient::ChannelState::AdsCallState::~AdsCallState() { - grpc_metadata_array_destroy(&initial_metadata_recv_); - grpc_metadata_array_destroy(&trailing_metadata_recv_); - grpc_byte_buffer_destroy(send_message_payload_); - grpc_byte_buffer_destroy(recv_message_payload_); - grpc_slice_unref_internal(status_details_); - GPR_ASSERT(call_ != nullptr); - grpc_call_unref(call_); } void XdsClient::ChannelState::AdsCallState::Orphan() { - GPR_ASSERT(call_ != nullptr); - // If we are here because xds_client wants to cancel the call, - // on_status_received_ will complete the cancellation and clean up. Otherwise, - // we are here because xds_client has to orphan a failed call, then the - // following cancellation will be a no-op. - grpc_call_cancel_internal(call_); state_map_.clear(); - // Note that the initial ref is hold by on_status_received_. So the - // corresponding unref happens in on_status_received_ instead of here. + // Note that the initial ref is held by the StreamEventHandler, which + // will be destroyed when call_ is destroyed, which may not happen + // here, since there may be other refs held to call_ by internal callbacks. + call_.reset(); } void XdsClient::ChannelState::AdsCallState::SendMessageLocked( const XdsResourceType* type) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_) { // Buffer message sending if an existing message is in flight. - if (send_message_payload_ != nullptr) { + if (send_message_pending_ != nullptr) { buffered_requests_.insert(type); return; } auto& state = state_map_[type]; - grpc_slice request_payload_slice; - request_payload_slice = xds_client()->api_.CreateAdsRequest( - chand()->server_, - chand()->server_.ShouldUseV3() ? type->type_url() : type->v2_type_url(), - chand()->resource_type_version_map_[type], state.nonce, - ResourceNamesForRequest(type), GRPC_ERROR_REF(state.error), - !sent_initial_message_); + std::string serialized_message = xds_client()->api_.CreateAdsRequest( + type->type_url(), chand()->resource_type_version_map_[type], state.nonce, + ResourceNamesForRequest(type), state.status, !sent_initial_message_); sent_initial_message_ = true; if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] xds server %s: sending ADS request: type=%s " "version=%s nonce=%s error=%s", - xds_client(), chand()->server_.server_uri.c_str(), + xds_client(), chand()->server_.server_uri().c_str(), std::string(type->type_url()).c_str(), chand()->resource_type_version_map_[type].c_str(), - state.nonce.c_str(), grpc_error_std_string(state.error).c_str()); - } - GRPC_ERROR_UNREF(state.error); - state.error = GRPC_ERROR_NONE; - // Create message payload. - send_message_payload_ = - grpc_raw_byte_buffer_create(&request_payload_slice, 1); - grpc_slice_unref_internal(request_payload_slice); - // Send the message. - grpc_op op; - memset(&op, 0, sizeof(op)); - op.op = GRPC_OP_SEND_MESSAGE; - op.data.send_message.send_message = send_message_payload_; - Ref(DEBUG_LOCATION, "ADS+OnRequestSentLocked").release(); - GRPC_CLOSURE_INIT(&on_request_sent_, OnRequestSent, this, - grpc_schedule_on_exec_ctx); - grpc_call_error call_error = - grpc_call_start_batch_and_execute(call_, &op, 1, &on_request_sent_); - if (GPR_UNLIKELY(call_error != GRPC_CALL_OK)) { - gpr_log(GPR_ERROR, - "[xds_client %p] xds server %s: error starting ADS send_message " - "batch on calld=%p: call_error=%d", - xds_client(), chand()->server_.server_uri.c_str(), this, - call_error); - GPR_ASSERT(GRPC_CALL_OK == call_error); + state.nonce.c_str(), state.status.ToString().c_str()); } + state.status = absl::OkStatus(); + call_->SendMessage(std::move(serialized_message)); + send_message_pending_ = type; } void XdsClient::ChannelState::AdsCallState::SubscribeLocked( @@ -1078,7 +987,12 @@ void XdsClient::ChannelState::AdsCallState::UnsubscribeLocked( if (authority_map.empty()) { type_state_map.subscribed_resources.erase(name.authority); } - if (!delay_unsubscription) SendMessageLocked(type); + // Don't need to send unsubscription message if this was the last + // resource we were subscribed to, since we'll be closing the stream + // immediately in that case. + if (!delay_unsubscription && HasSubscribedResources()) { + SendMessageLocked(type); + } } bool XdsClient::ChannelState::AdsCallState::HasSubscribedResources() const { @@ -1088,22 +1002,21 @@ bool XdsClient::ChannelState::AdsCallState::HasSubscribedResources() const { return false; } -void XdsClient::ChannelState::AdsCallState::OnRequestSent( - void* arg, grpc_error_handle error) { - AdsCallState* ads_calld = static_cast<AdsCallState*>(arg); - { - MutexLock lock(&ads_calld->xds_client()->mu_); - ads_calld->OnRequestSentLocked(GRPC_ERROR_REF(error)); +void XdsClient::ChannelState::AdsCallState::OnRequestSent(bool ok) { + MutexLock lock(&xds_client()->mu_); + // For each resource that was in the message we just sent, start the + // resource timer if needed. + if (ok) { + auto& resource_type_state = state_map_[send_message_pending_]; + for (const auto& p : resource_type_state.subscribed_resources) { + for (auto& q : p.second) { + q.second->MaybeMarkSubscriptionSendComplete( + Ref(DEBUG_LOCATION, "ResourceTimer")); + } + } } - ads_calld->Unref(DEBUG_LOCATION, "ADS+OnRequestSentLocked"); -} - -void XdsClient::ChannelState::AdsCallState::OnRequestSentLocked( - grpc_error_handle error) { - if (IsCurrentCallOnChannel() && error == GRPC_ERROR_NONE) { - // Clean up the sent message. - grpc_byte_buffer_destroy(send_message_payload_); - send_message_payload_ = nullptr; + send_message_pending_ = nullptr; + if (ok && IsCurrentCallOnChannel()) { // Continue to send another pending message if any. // TODO(roth): The current code to handle buffered messages has the // advantage of sending only the most recent list of resource names for @@ -1119,170 +1032,149 @@ void XdsClient::ChannelState::AdsCallState::OnRequestSentLocked( buffered_requests_.erase(it); } } - GRPC_ERROR_UNREF(error); } -void XdsClient::ChannelState::AdsCallState::OnResponseReceived( - void* arg, grpc_error_handle /* error */) { - AdsCallState* ads_calld = static_cast<AdsCallState*>(arg); - bool done; +void XdsClient::ChannelState::AdsCallState::OnRecvMessage( + absl::string_view payload) { { - MutexLock lock(&ads_calld->xds_client()->mu_); - done = ads_calld->OnResponseReceivedLocked(); - } - ads_calld->xds_client()->work_serializer_.DrainQueue(); - if (done) ads_calld->Unref(DEBUG_LOCATION, "ADS+OnResponseReceivedLocked"); -} - -bool XdsClient::ChannelState::AdsCallState::OnResponseReceivedLocked() { - // Empty payload means the call was cancelled. - if (!IsCurrentCallOnChannel() || recv_message_payload_ == nullptr) { - return true; - } - // Read the response. - grpc_byte_buffer_reader bbr; - grpc_byte_buffer_reader_init(&bbr, recv_message_payload_); - grpc_slice response_slice = grpc_byte_buffer_reader_readall(&bbr); - grpc_byte_buffer_reader_destroy(&bbr); - grpc_byte_buffer_destroy(recv_message_payload_); - recv_message_payload_ = nullptr; - // Parse and validate the response. - AdsResponseParser parser(this); - absl::Status status = xds_client()->api_.ParseAdsResponse( - chand()->server_, response_slice, &parser); - grpc_slice_unref_internal(response_slice); - if (!status.ok()) { - // Ignore unparsable response. - gpr_log(GPR_ERROR, - "[xds_client %p] xds server %s: error parsing ADS response (%s) " - "-- ignoring", - xds_client(), chand()->server_.server_uri.c_str(), - status.ToString().c_str()); - } else { - seen_response_ = true; - AdsResponseParser::Result result = parser.TakeResult(); - // Update nonce. - auto& state = state_map_[result.type]; - state.nonce = result.nonce; - // If we got an error, set state.error so that we'll NACK the update. - if (!result.errors.empty()) { - std::string error = absl::StrJoin(result.errors, "; "); - gpr_log( - GPR_ERROR, - "[xds_client %p] xds server %s: ADS response invalid for resource " - "type %s version %s, will NACK: nonce=%s error=%s", - xds_client(), chand()->server_.server_uri.c_str(), - result.type_url.c_str(), result.version.c_str(), state.nonce.c_str(), - error.c_str()); - GRPC_ERROR_UNREF(state.error); - state.error = grpc_error_set_int(GRPC_ERROR_CREATE_FROM_CPP_STRING(error), - GRPC_ERROR_INT_GRPC_STATUS, - GRPC_STATUS_UNAVAILABLE); - } - // Delete resources not seen in update if needed. - if (result.type->AllResourcesRequiredInSotW()) { - for (auto& a : xds_client()->authority_state_map_) { - const std::string& authority = a.first; - AuthorityState& authority_state = a.second; - // Skip authorities that are not using this xDS channel. - if (authority_state.channel_state != chand()) continue; - auto seen_authority_it = result.resources_seen.find(authority); - // Find this resource type. - auto type_it = authority_state.resource_map.find(result.type); - if (type_it == authority_state.resource_map.end()) continue; - // Iterate over resource ids. - for (auto& r : type_it->second) { - const XdsResourceKey& resource_key = r.first; - ResourceState& resource_state = r.second; - if (seen_authority_it == result.resources_seen.end() || - seen_authority_it->second.find(resource_key) == - seen_authority_it->second.end()) { - // If the resource was newly requested but has not yet been - // received, we don't want to generate an error for the watchers, - // because this ADS response may be in reaction to an earlier - // request that did not yet request the new resource, so its absence - // from the response does not necessarily indicate that the resource - // does not exist. For that case, we rely on the request timeout - // instead. - if (resource_state.resource == nullptr) continue; - resource_state.resource.reset(); - xds_client()->NotifyWatchersOnResourceDoesNotExist( - resource_state.watchers); + MutexLock lock(&xds_client()->mu_); + if (!IsCurrentCallOnChannel()) return; + // Parse and validate the response. + AdsResponseParser parser(this); + absl::Status status = xds_client()->api_.ParseAdsResponse(payload, &parser); + if (!status.ok()) { + // Ignore unparsable response. + gpr_log(GPR_ERROR, + "[xds_client %p] xds server %s: error parsing ADS response (%s) " + "-- ignoring", + xds_client(), chand()->server_.server_uri().c_str(), + status.ToString().c_str()); + } else { + seen_response_ = true; + chand()->status_ = absl::OkStatus(); + AdsResponseParser::Result result = parser.TakeResult(); + // Update nonce. + auto& state = state_map_[result.type]; + state.nonce = result.nonce; + // If we got an error, set state.status so that we'll NACK the update. + if (!result.errors.empty()) { + state.status = absl::UnavailableError( + absl::StrCat("xDS response validation errors: [", + absl::StrJoin(result.errors, "; "), "]")); + gpr_log(GPR_ERROR, + "[xds_client %p] xds server %s: ADS response invalid for " + "resource " + "type %s version %s, will NACK: nonce=%s status=%s", + xds_client(), chand()->server_.server_uri().c_str(), + result.type_url.c_str(), result.version.c_str(), + state.nonce.c_str(), state.status.ToString().c_str()); + } + // Delete resources not seen in update if needed. + if (result.type->AllResourcesRequiredInSotW()) { + for (auto& a : xds_client()->authority_state_map_) { + const std::string& authority = a.first; + AuthorityState& authority_state = a.second; + // Skip authorities that are not using this xDS channel. + if (authority_state.channel_state != chand()) continue; + auto seen_authority_it = result.resources_seen.find(authority); + // Find this resource type. + auto type_it = authority_state.resource_map.find(result.type); + if (type_it == authority_state.resource_map.end()) continue; + // Iterate over resource ids. + for (auto& r : type_it->second) { + const XdsResourceKey& resource_key = r.first; + ResourceState& resource_state = r.second; + if (seen_authority_it == result.resources_seen.end() || + seen_authority_it->second.find(resource_key) == + seen_authority_it->second.end()) { + // If the resource was newly requested but has not yet been + // received, we don't want to generate an error for the + // watchers, because this ADS response may be in reaction to an + // earlier request that did not yet request the new resource, so + // its absence from the response does not necessarily indicate + // that the resource does not exist. For that case, we rely on + // the request timeout instead. + if (resource_state.resource == nullptr) continue; + if (chand()->server_.IgnoreResourceDeletion()) { + if (!resource_state.ignored_deletion) { + gpr_log(GPR_ERROR, + "[xds_client %p] xds server %s: ignoring deletion " + "for resource type %s name %s", + xds_client(), chand()->server_.server_uri().c_str(), + result.type_url.c_str(), + XdsClient::ConstructFullXdsResourceName( + authority, result.type_url.c_str(), resource_key) + .c_str()); + resource_state.ignored_deletion = true; + } + } else { + resource_state.resource.reset(); + resource_state.meta.client_status = + XdsApi::ResourceMetadata::DOES_NOT_EXIST; + xds_client()->NotifyWatchersOnResourceDoesNotExist( + resource_state.watchers); + } + } } } } - } - // If we had valid resources, update the version. - if (result.have_valid_resources) { - chand()->resource_type_version_map_[result.type] = - std::move(result.version); - // Start load reporting if needed. - auto& lrs_call = chand()->lrs_calld_; - if (lrs_call != nullptr) { - LrsCallState* lrs_calld = lrs_call->calld(); - if (lrs_calld != nullptr) lrs_calld->MaybeStartReportingLocked(); + // If we had valid resources or the update was empty, update the version. + if (result.have_valid_resources || result.errors.empty()) { + chand()->resource_type_version_map_[result.type] = + std::move(result.version); + // Start load reporting if needed. + auto& lrs_call = chand()->lrs_calld_; + if (lrs_call != nullptr) { + LrsCallState* lrs_calld = lrs_call->calld(); + if (lrs_calld != nullptr) lrs_calld->MaybeStartReportingLocked(); + } } + // Send ACK or NACK. + SendMessageLocked(result.type); } - // Send ACK or NACK. - SendMessageLocked(result.type); - } - if (xds_client()->shutting_down_) return true; - // Keep listening for updates. - grpc_op op; - memset(&op, 0, sizeof(op)); - op.op = GRPC_OP_RECV_MESSAGE; - op.data.recv_message.recv_message = &recv_message_payload_; - op.flags = 0; - op.reserved = nullptr; - GPR_ASSERT(call_ != nullptr); - // Reuse the "ADS+OnResponseReceivedLocked" ref taken in ctor. - const grpc_call_error call_error = - grpc_call_start_batch_and_execute(call_, &op, 1, &on_response_received_); - GPR_ASSERT(GRPC_CALL_OK == call_error); - return false; + } + xds_client()->work_serializer_.DrainQueue(); } void XdsClient::ChannelState::AdsCallState::OnStatusReceived( - void* arg, grpc_error_handle error) { - AdsCallState* ads_calld = static_cast<AdsCallState*>(arg); + absl::Status status) { { - MutexLock lock(&ads_calld->xds_client()->mu_); - ads_calld->OnStatusReceivedLocked(GRPC_ERROR_REF(error)); - } - ads_calld->xds_client()->work_serializer_.DrainQueue(); - ads_calld->Unref(DEBUG_LOCATION, "ADS+OnStatusReceivedLocked"); -} - -void XdsClient::ChannelState::AdsCallState::OnStatusReceivedLocked( - grpc_error_handle error) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - char* status_details = grpc_slice_to_c_string(status_details_); - gpr_log(GPR_INFO, - "[xds_client %p] xds server %s: ADS call status received " - "(chand=%p, ads_calld=%p, call=%p): " - "status=%d, details='%s', error='%s'", - xds_client(), chand()->server_.server_uri.c_str(), chand(), this, - call_, status_code_, status_details, - grpc_error_std_string(error).c_str()); - gpr_free(status_details); - } - // Ignore status from a stale call. - if (IsCurrentCallOnChannel()) { - // Try to restart the call. - parent_->OnCallFinishedLocked(); - // Send error to all watchers. - xds_client()->NotifyOnErrorLocked(absl::UnavailableError(absl::StrFormat( - "xDS call failed: xDS server: %s, ADS call status code=%d, " - "details='%s', error='%s'", - chand()->server_.server_uri, status_code_, - StringViewFromSlice(status_details_), grpc_error_std_string(error)))); + MutexLock lock(&xds_client()->mu_); + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, + "[xds_client %p] xds server %s: ADS call status received " + "(chand=%p, ads_calld=%p, call=%p): %s", + xds_client(), chand()->server_.server_uri().c_str(), chand(), + this, call_.get(), status.ToString().c_str()); + } + // Cancel any does-not-exist timers that may be pending. + for (const auto& p : state_map_) { + for (const auto& q : p.second.subscribed_resources) { + for (auto& r : q.second) { + r.second->MaybeCancelTimer(); + } + } + } + // Ignore status from a stale call. + if (IsCurrentCallOnChannel()) { + // Try to restart the call. + parent_->OnCallFinishedLocked(); + // If we didn't receive a response on the stream, report the + // stream failure as a connectivity failure, which will report the + // error to all watchers of resources on this channel. + if (!seen_response_) { + chand()->SetChannelStatusLocked(absl::UnavailableError( + absl::StrCat("xDS call failed with no responses received; status: ", + status.ToString()))); + } + } } - GRPC_ERROR_UNREF(error); + xds_client()->work_serializer_.DrainQueue(); } bool XdsClient::ChannelState::AdsCallState::IsCurrentCallOnChannel() const { - // If the retryable ADS call is null (which only happens when the xds channel - // is shutting down), all the ADS calls are stale. + // If the retryable ADS call is null (which only happens when the xds + // channel is shutting down), all the ADS calls are stale. if (chand()->ads_calld_ == nullptr) return false; return this == chand()->ads_calld_->calld(); } @@ -1300,7 +1192,7 @@ XdsClient::ChannelState::AdsCallState::ResourceNamesForRequest( resource_names.emplace_back(XdsClient::ConstructFullXdsResourceName( authority, type->type_url(), resource_key)); OrphanablePtr<ResourceTimer>& resource_timer = p.second; - resource_timer->MaybeStartTimer(Ref(DEBUG_LOCATION, "ResourceTimer")); + resource_timer->MarkSubscriptionSendStarted(); } } } @@ -1312,38 +1204,35 @@ XdsClient::ChannelState::AdsCallState::ResourceNamesForRequest( // void XdsClient::ChannelState::LrsCallState::Reporter::Orphan() { - if (next_report_timer_callback_pending_) { - grpc_timer_cancel(&next_report_timer_); + if (timer_handle_.has_value() && + xds_client()->engine()->Cancel(*timer_handle_)) { + timer_handle_.reset(); + Unref(DEBUG_LOCATION, "Orphan"); } } void XdsClient::ChannelState::LrsCallState::Reporter:: ScheduleNextReportLocked() { - const Timestamp next_report_time = ExecCtx::Get()->Now() + report_interval_; - grpc_timer_init(&next_report_timer_, next_report_time, - &on_next_report_timer_); - next_report_timer_callback_pending_ = true; -} - -void XdsClient::ChannelState::LrsCallState::Reporter::OnNextReportTimer( - void* arg, grpc_error_handle error) { - Reporter* self = static_cast<Reporter*>(arg); - bool done; - { - MutexLock lock(&self->xds_client()->mu_); - done = self->OnNextReportTimerLocked(GRPC_ERROR_REF(error)); - } - if (done) self->Unref(DEBUG_LOCATION, "Reporter+timer"); + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, + "[xds_client %p] xds server %s: scheduling load report timer", + xds_client(), parent_->chand()->server_.server_uri().c_str()); + } + timer_handle_ = xds_client()->engine()->RunAfter(report_interval_, [this]() { + ApplicationCallbackExecCtx callback_exec_ctx; + ExecCtx exec_ctx; + if (OnNextReportTimer()) { + Unref(DEBUG_LOCATION, "OnNextReportTimer()"); + } + }); } -bool XdsClient::ChannelState::LrsCallState::Reporter::OnNextReportTimerLocked( - grpc_error_handle error) { - next_report_timer_callback_pending_ = false; - if (error != GRPC_ERROR_NONE || !IsCurrentReporterOnCall()) { - GRPC_ERROR_UNREF(error); - return true; - } - return SendReportLocked(); +bool XdsClient::ChannelState::LrsCallState::Reporter::OnNextReportTimer() { + MutexLock lock(&xds_client()->mu_); + timer_handle_.reset(); + if (!IsCurrentReporterOnCall()) return true; + SendReportLocked(); + return false; } namespace { @@ -1374,7 +1263,7 @@ bool XdsClient::ChannelState::LrsCallState::Reporter::SendReportLocked() { last_report_counters_were_zero_ = LoadReportCountersAreZero(snapshot); if (old_val && last_report_counters_were_zero_) { auto it = xds_client()->xds_load_report_server_map_.find( - parent_->chand()->server_); + &parent_->chand()->server_); if (it == xds_client()->xds_load_report_server_map_.end() || it->second.load_report_map.empty()) { it->second.channel_state->StopLrsCallLocked(); @@ -1383,65 +1272,34 @@ bool XdsClient::ChannelState::LrsCallState::Reporter::SendReportLocked() { ScheduleNextReportLocked(); return false; } - // Create a request that contains the snapshot. - grpc_slice request_payload_slice = + // Send a request that contains the snapshot. + std::string serialized_payload = xds_client()->api_.CreateLrsRequest(std::move(snapshot)); - parent_->send_message_payload_ = - grpc_raw_byte_buffer_create(&request_payload_slice, 1); - grpc_slice_unref_internal(request_payload_slice); - // Send the report. - grpc_op op; - memset(&op, 0, sizeof(op)); - op.op = GRPC_OP_SEND_MESSAGE; - op.data.send_message.send_message = parent_->send_message_payload_; - grpc_call_error call_error = grpc_call_start_batch_and_execute( - parent_->call_, &op, 1, &on_report_done_); - if (GPR_UNLIKELY(call_error != GRPC_CALL_OK)) { - gpr_log(GPR_ERROR, - "[xds_client %p] xds server %s: error starting LRS send_message " - "batch on calld=%p: call_error=%d", - xds_client(), parent_->chand()->server_.server_uri.c_str(), this, - call_error); - GPR_ASSERT(GRPC_CALL_OK == call_error); - } + parent_->call_->SendMessage(std::move(serialized_payload)); + parent_->send_message_pending_ = true; return false; } -void XdsClient::ChannelState::LrsCallState::Reporter::OnReportDone( - void* arg, grpc_error_handle error) { - Reporter* self = static_cast<Reporter*>(arg); - bool done; - { - MutexLock lock(&self->xds_client()->mu_); - done = self->OnReportDoneLocked(GRPC_ERROR_REF(error)); - } - if (done) self->Unref(DEBUG_LOCATION, "Reporter+report_done"); -} - -bool XdsClient::ChannelState::LrsCallState::Reporter::OnReportDoneLocked( - grpc_error_handle error) { - grpc_byte_buffer_destroy(parent_->send_message_payload_); - parent_->send_message_payload_ = nullptr; +void XdsClient::ChannelState::LrsCallState::Reporter::OnReportDoneLocked() { + // If a reporter starts a send_message op, then the reporting interval + // changes and we destroy that reporter and create a new one, and then + // the send_message op started by the old reporter finishes, this + // method will be called even though it was for a completion started + // by the old reporter. In that case, the timer will be pending, so + // we just ignore the completion and wait for the timer to fire. + if (timer_handle_.has_value()) return; // If there are no more registered stats to report, cancel the call. - auto it = - xds_client()->xds_load_report_server_map_.find(parent_->chand()->server_); - if (it == xds_client()->xds_load_report_server_map_.end() || - it->second.load_report_map.empty()) { - it->second.channel_state->StopLrsCallLocked(); - GRPC_ERROR_UNREF(error); - return true; - } - if (error != GRPC_ERROR_NONE || !IsCurrentReporterOnCall()) { - GRPC_ERROR_UNREF(error); - // If this reporter is no longer the current one on the call, the reason - // might be that it was orphaned for a new one due to config update. - if (!IsCurrentReporterOnCall()) { - parent_->MaybeStartReportingLocked(); + auto it = xds_client()->xds_load_report_server_map_.find( + &parent_->chand()->server_); + if (it == xds_client()->xds_load_report_server_map_.end()) return; + if (it->second.load_report_map.empty()) { + if (it->second.channel_state != nullptr) { + it->second.channel_state->StopLrsCallLocked(); } - return true; + return; } + // Otherwise, schedule the next load report. ScheduleNextReportLocked(); - return false; } // @@ -1460,123 +1318,41 @@ XdsClient::ChannelState::LrsCallState::LrsCallState( // the polling entities from client_channel. GPR_ASSERT(xds_client() != nullptr); const char* method = - chand()->server_.ShouldUseV3() - ? "/envoy.service.load_stats.v3.LoadReportingService/StreamLoadStats" - : "/envoy.service.load_stats.v2.LoadReportingService/StreamLoadStats"; - call_ = grpc_channel_create_pollset_set_call( - chand()->channel_, nullptr, GRPC_PROPAGATE_DEFAULTS, - xds_client()->interested_parties_, - Slice::FromStaticString(method).c_slice(), nullptr, - Timestamp::InfFuture(), nullptr); + "/envoy.service.load_stats.v3.LoadReportingService/StreamLoadStats"; + call_ = chand()->transport_->CreateStreamingCall( + method, std::make_unique<StreamEventHandler>( + // Passing the initial ref here. This ref will go away when + // the StreamEventHandler is destroyed. + RefCountedPtr<LrsCallState>(this))); GPR_ASSERT(call_ != nullptr); - // Init the request payload. - grpc_slice request_payload_slice = - xds_client()->api_.CreateLrsInitialRequest(chand()->server_); - send_message_payload_ = - grpc_raw_byte_buffer_create(&request_payload_slice, 1); - grpc_slice_unref_internal(request_payload_slice); - // Init other data associated with the LRS call. - grpc_metadata_array_init(&initial_metadata_recv_); - grpc_metadata_array_init(&trailing_metadata_recv_); // Start the call. if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log( - GPR_INFO, - "[xds_client %p] xds server %s: starting LRS call (calld=%p, call=%p)", - xds_client(), chand()->server_.server_uri.c_str(), this, call_); - } - // Create the ops. - grpc_call_error call_error; - grpc_op ops[3]; - memset(ops, 0, sizeof(ops)); - // Op: send initial metadata. - grpc_op* op = ops; - op->op = GRPC_OP_SEND_INITIAL_METADATA; - op->data.send_initial_metadata.count = 0; - op->flags = GRPC_INITIAL_METADATA_WAIT_FOR_READY | - GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET; - op->reserved = nullptr; - op++; - // Op: send request message. - GPR_ASSERT(send_message_payload_ != nullptr); - op->op = GRPC_OP_SEND_MESSAGE; - op->data.send_message.send_message = send_message_payload_; - op->flags = 0; - op->reserved = nullptr; - op++; - Ref(DEBUG_LOCATION, "LRS+OnInitialRequestSentLocked").release(); - GRPC_CLOSURE_INIT(&on_initial_request_sent_, OnInitialRequestSent, this, - grpc_schedule_on_exec_ctx); - call_error = grpc_call_start_batch_and_execute( - call_, ops, static_cast<size_t>(op - ops), &on_initial_request_sent_); - GPR_ASSERT(GRPC_CALL_OK == call_error); - // Op: recv initial metadata. - op = ops; - op->op = GRPC_OP_RECV_INITIAL_METADATA; - op->data.recv_initial_metadata.recv_initial_metadata = - &initial_metadata_recv_; - op->flags = 0; - op->reserved = nullptr; - op++; - // Op: recv response. - op->op = GRPC_OP_RECV_MESSAGE; - op->data.recv_message.recv_message = &recv_message_payload_; - op->flags = 0; - op->reserved = nullptr; - op++; - Ref(DEBUG_LOCATION, "LRS+OnResponseReceivedLocked").release(); - GRPC_CLOSURE_INIT(&on_response_received_, OnResponseReceived, this, - grpc_schedule_on_exec_ctx); - call_error = grpc_call_start_batch_and_execute( - call_, ops, static_cast<size_t>(op - ops), &on_response_received_); - GPR_ASSERT(GRPC_CALL_OK == call_error); - // Op: recv server status. - op = ops; - op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; - op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv_; - op->data.recv_status_on_client.status = &status_code_; - op->data.recv_status_on_client.status_details = &status_details_; - op->flags = 0; - op->reserved = nullptr; - op++; - // This callback signals the end of the call, so it relies on the initial - // ref instead of a new ref. When it's invoked, it's the initial ref that is - // unreffed. - GRPC_CLOSURE_INIT(&on_status_received_, OnStatusReceived, this, - grpc_schedule_on_exec_ctx); - call_error = grpc_call_start_batch_and_execute( - call_, ops, static_cast<size_t>(op - ops), &on_status_received_); - GPR_ASSERT(GRPC_CALL_OK == call_error); -} - -XdsClient::ChannelState::LrsCallState::~LrsCallState() { - grpc_metadata_array_destroy(&initial_metadata_recv_); - grpc_metadata_array_destroy(&trailing_metadata_recv_); - grpc_byte_buffer_destroy(send_message_payload_); - grpc_byte_buffer_destroy(recv_message_payload_); - grpc_slice_unref_internal(status_details_); - GPR_ASSERT(call_ != nullptr); - grpc_call_unref(call_); + gpr_log(GPR_INFO, + "[xds_client %p] xds server %s: starting LRS call (calld=%p, " + "call=%p)", + xds_client(), chand()->server_.server_uri().c_str(), this, + call_.get()); + } + // Send the initial request. + std::string serialized_payload = xds_client()->api_.CreateLrsInitialRequest(); + call_->SendMessage(std::move(serialized_payload)); + send_message_pending_ = true; } void XdsClient::ChannelState::LrsCallState::Orphan() { reporter_.reset(); - GPR_ASSERT(call_ != nullptr); - // If we are here because xds_client wants to cancel the call, - // on_status_received_ will complete the cancellation and clean up. Otherwise, - // we are here because xds_client has to orphan a failed call, then the - // following cancellation will be a no-op. - grpc_call_cancel_internal(call_); - // Note that the initial ref is hold by on_status_received_. So the - // corresponding unref happens in on_status_received_ instead of here. + // Note that the initial ref is held by the StreamEventHandler, which + // will be destroyed when call_ is destroyed, which may not happen + // here, since there may be other refs held to call_ by internal callbacks. + call_.reset(); } void XdsClient::ChannelState::LrsCallState::MaybeStartReportingLocked() { // Don't start again if already started. if (reporter_ != nullptr) return; - // Don't start if the previous send_message op (of the initial request or the - // last report of the previous reporter) hasn't completed. - if (send_message_payload_ != nullptr) return; + // Don't start if the previous send_message op (of the initial request or + // the last report of the previous reporter) hasn't completed. + if (call_ != nullptr && send_message_pending_) return; // Don't start if no LRS response has arrived. if (!seen_response()) return; // Don't start if the ADS call hasn't received any valid response. Note that @@ -1588,170 +1364,113 @@ void XdsClient::ChannelState::LrsCallState::MaybeStartReportingLocked() { return; } // Start reporting. + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, "[xds_client %p] xds server %s: creating load reporter", + xds_client(), chand()->server_.server_uri().c_str()); + } reporter_ = MakeOrphanable<Reporter>( Ref(DEBUG_LOCATION, "LRS+load_report+start"), load_reporting_interval_); } -void XdsClient::ChannelState::LrsCallState::OnInitialRequestSent( - void* arg, grpc_error_handle /*error*/) { - LrsCallState* lrs_calld = static_cast<LrsCallState*>(arg); - { - MutexLock lock(&lrs_calld->xds_client()->mu_); - lrs_calld->OnInitialRequestSentLocked(); +void XdsClient::ChannelState::LrsCallState::OnRequestSent(bool /*ok*/) { + MutexLock lock(&xds_client()->mu_); + send_message_pending_ = false; + if (reporter_ != nullptr) { + reporter_->OnReportDoneLocked(); + } else { + MaybeStartReportingLocked(); } - lrs_calld->Unref(DEBUG_LOCATION, "LRS+OnInitialRequestSentLocked"); } -void XdsClient::ChannelState::LrsCallState::OnInitialRequestSentLocked() { - // Clear the send_message_payload_. - grpc_byte_buffer_destroy(send_message_payload_); - send_message_payload_ = nullptr; - MaybeStartReportingLocked(); -} - -void XdsClient::ChannelState::LrsCallState::OnResponseReceived( - void* arg, grpc_error_handle /*error*/) { - LrsCallState* lrs_calld = static_cast<LrsCallState*>(arg); - bool done; - { - MutexLock lock(&lrs_calld->xds_client()->mu_); - done = lrs_calld->OnResponseReceivedLocked(); - } - if (done) lrs_calld->Unref(DEBUG_LOCATION, "LRS+OnResponseReceivedLocked"); -} - -bool XdsClient::ChannelState::LrsCallState::OnResponseReceivedLocked() { - // Empty payload means the call was cancelled. - if (!IsCurrentCallOnChannel() || recv_message_payload_ == nullptr) { - return true; - } - // Read the response. - grpc_byte_buffer_reader bbr; - grpc_byte_buffer_reader_init(&bbr, recv_message_payload_); - grpc_slice response_slice = grpc_byte_buffer_reader_readall(&bbr); - grpc_byte_buffer_reader_destroy(&bbr); - grpc_byte_buffer_destroy(recv_message_payload_); - recv_message_payload_ = nullptr; - // This anonymous lambda is a hack to avoid the usage of goto. - [&]() { - // Parse the response. - bool send_all_clusters = false; - std::set<std::string> new_cluster_names; - Duration new_load_reporting_interval; - grpc_error_handle parse_error = xds_client()->api_.ParseLrsResponse( - response_slice, &send_all_clusters, &new_cluster_names, - &new_load_reporting_interval); - if (parse_error != GRPC_ERROR_NONE) { - gpr_log(GPR_ERROR, - "[xds_client %p] xds server %s: LRS response parsing failed: %s", - xds_client(), chand()->server_.server_uri.c_str(), - grpc_error_std_string(parse_error).c_str()); - GRPC_ERROR_UNREF(parse_error); - return; +void XdsClient::ChannelState::LrsCallState::OnRecvMessage( + absl::string_view payload) { + MutexLock lock(&xds_client()->mu_); + // If we're no longer the current call, ignore the result. + if (!IsCurrentCallOnChannel()) return; + // Parse the response. + bool send_all_clusters = false; + std::set<std::string> new_cluster_names; + Duration new_load_reporting_interval; + absl::Status status = xds_client()->api_.ParseLrsResponse( + payload, &send_all_clusters, &new_cluster_names, + &new_load_reporting_interval); + if (!status.ok()) { + gpr_log(GPR_ERROR, + "[xds_client %p] xds server %s: LRS response parsing failed: %s", + xds_client(), chand()->server_.server_uri().c_str(), + status.ToString().c_str()); + return; + } + seen_response_ = true; + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log( + GPR_INFO, + "[xds_client %p] xds server %s: LRS response received, %" PRIuPTR + " cluster names, send_all_clusters=%d, load_report_interval=%" PRId64 + "ms", + xds_client(), chand()->server_.server_uri().c_str(), + new_cluster_names.size(), send_all_clusters, + new_load_reporting_interval.millis()); + size_t i = 0; + for (const auto& name : new_cluster_names) { + gpr_log(GPR_INFO, "[xds_client %p] cluster_name %" PRIuPTR ": %s", + xds_client(), i++, name.c_str()); } - seen_response_ = true; + } + if (new_load_reporting_interval < + Duration::Milliseconds(GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS)) { + new_load_reporting_interval = + Duration::Milliseconds(GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log( - GPR_INFO, - "[xds_client %p] xds server %s: LRS response received, %" PRIuPTR - " cluster names, send_all_clusters=%d, load_report_interval=%" PRId64 - "ms", - xds_client(), chand()->server_.server_uri.c_str(), - new_cluster_names.size(), send_all_clusters, - new_load_reporting_interval.millis()); - size_t i = 0; - for (const auto& name : new_cluster_names) { - gpr_log(GPR_INFO, "[xds_client %p] cluster_name %" PRIuPTR ": %s", - xds_client(), i++, name.c_str()); - } - } - if (new_load_reporting_interval < - Duration::Milliseconds( - GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS)) { - new_load_reporting_interval = Duration::Milliseconds( - GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS); - if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log(GPR_INFO, - "[xds_client %p] xds server %s: increased load_report_interval " - "to minimum value %dms", - xds_client(), chand()->server_.server_uri.c_str(), - GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS); - } + gpr_log(GPR_INFO, + "[xds_client %p] xds server %s: increased load_report_interval " + "to minimum value %dms", + xds_client(), chand()->server_.server_uri().c_str(), + GRPC_XDS_MIN_CLIENT_LOAD_REPORTING_INTERVAL_MS); } - // Ignore identical update. - if (send_all_clusters == send_all_clusters_ && - cluster_names_ == new_cluster_names && - load_reporting_interval_ == new_load_reporting_interval) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log( - GPR_INFO, - "[xds_client %p] xds server %s: incoming LRS response identical " - "to current, ignoring.", - xds_client(), chand()->server_.server_uri.c_str()); - } - return; + } + // Ignore identical update. + if (send_all_clusters == send_all_clusters_ && + cluster_names_ == new_cluster_names && + load_reporting_interval_ == new_load_reporting_interval) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, + "[xds_client %p] xds server %s: incoming LRS response identical " + "to current, ignoring.", + xds_client(), chand()->server_.server_uri().c_str()); } - // Stop current load reporting (if any) to adopt the new config. - reporter_.reset(); - // Record the new config. - send_all_clusters_ = send_all_clusters; - cluster_names_ = std::move(new_cluster_names); - load_reporting_interval_ = new_load_reporting_interval; - // Try starting sending load report. - MaybeStartReportingLocked(); - }(); - grpc_slice_unref_internal(response_slice); - if (xds_client()->shutting_down_) return true; - // Keep listening for LRS config updates. - grpc_op op; - memset(&op, 0, sizeof(op)); - op.op = GRPC_OP_RECV_MESSAGE; - op.data.recv_message.recv_message = &recv_message_payload_; - op.flags = 0; - op.reserved = nullptr; - GPR_ASSERT(call_ != nullptr); - // Reuse the "OnResponseReceivedLocked" ref taken in ctor. - const grpc_call_error call_error = - grpc_call_start_batch_and_execute(call_, &op, 1, &on_response_received_); - GPR_ASSERT(GRPC_CALL_OK == call_error); - return false; -} - -void XdsClient::ChannelState::LrsCallState::OnStatusReceived( - void* arg, grpc_error_handle error) { - LrsCallState* lrs_calld = static_cast<LrsCallState*>(arg); - { - MutexLock lock(&lrs_calld->xds_client()->mu_); - lrs_calld->OnStatusReceivedLocked(GRPC_ERROR_REF(error)); + return; } - lrs_calld->Unref(DEBUG_LOCATION, "LRS+OnStatusReceivedLocked"); + // Stop current load reporting (if any) to adopt the new config. + reporter_.reset(); + // Record the new config. + send_all_clusters_ = send_all_clusters; + cluster_names_ = std::move(new_cluster_names); + load_reporting_interval_ = new_load_reporting_interval; + // Try starting sending load report. + MaybeStartReportingLocked(); } -void XdsClient::ChannelState::LrsCallState::OnStatusReceivedLocked( - grpc_error_handle error) { - GPR_ASSERT(call_ != nullptr); +void XdsClient::ChannelState::LrsCallState::OnStatusReceived( + absl::Status status) { + MutexLock lock(&xds_client()->mu_); if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - char* status_details = grpc_slice_to_c_string(status_details_); gpr_log(GPR_INFO, "[xds_client %p] xds server %s: LRS call status received " - "(chand=%p, calld=%p, call=%p): " - "status=%d, details='%s', error='%s'", - xds_client(), chand()->server_.server_uri.c_str(), chand(), this, - call_, status_code_, status_details, - grpc_error_std_string(error).c_str()); - gpr_free(status_details); + "(chand=%p, calld=%p, call=%p): %s", + xds_client(), chand()->server_.server_uri().c_str(), chand(), this, + call_.get(), status.ToString().c_str()); } // Ignore status from a stale call. if (IsCurrentCallOnChannel()) { // Try to restart the call. parent_->OnCallFinishedLocked(); } - GRPC_ERROR_UNREF(error); } bool XdsClient::ChannelState::LrsCallState::IsCurrentCallOnChannel() const { - // If the retryable LRS call is null (which only happens when the xds channel - // is shutting down), all the LRS calls are stale. + // If the retryable LRS call is null (which only happens when the xds + // channel is shutting down), all the LRS calls are stale. if (chand()->lrs_calld_ == nullptr) return false; return this == chand()->lrs_calld_->calld(); } @@ -1760,86 +1479,66 @@ bool XdsClient::ChannelState::LrsCallState::IsCurrentCallOnChannel() const { // XdsClient // -namespace { - -Duration GetRequestTimeout(const grpc_channel_args* args) { - return Duration::Milliseconds(grpc_channel_args_find_integer( - args, GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS, - {15000, 0, INT_MAX})); -} - -grpc_channel_args* ModifyChannelArgs(const grpc_channel_args* args) { - absl::InlinedVector<grpc_arg, 1> args_to_add = { - grpc_channel_arg_integer_create( - const_cast<char*>(GRPC_ARG_KEEPALIVE_TIME_MS), - 5 * 60 * GPR_MS_PER_SEC), - }; - return grpc_channel_args_copy_and_add(args, args_to_add.data(), - args_to_add.size()); -} - -} // namespace - -XdsClient::XdsClient(std::unique_ptr<XdsBootstrap> bootstrap, - const grpc_channel_args* args) +XdsClient::XdsClient( + std::unique_ptr<XdsBootstrap> bootstrap, + OrphanablePtr<XdsTransportFactory> transport_factory, + std::shared_ptr<grpc_event_engine::experimental::EventEngine> engine, + std::string user_agent_name, std::string user_agent_version, + Duration resource_request_timeout) : DualRefCounted<XdsClient>( GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_refcount_trace) ? "XdsClient" : nullptr), bootstrap_(std::move(bootstrap)), - args_(ModifyChannelArgs(args)), - request_timeout_(GetRequestTimeout(args)), + transport_factory_(std::move(transport_factory)), + request_timeout_(resource_request_timeout), xds_federation_enabled_(XdsFederationEnabled()), - interested_parties_(grpc_pollset_set_create()), - certificate_provider_store_(MakeOrphanable<CertificateProviderStore>( - bootstrap_->certificate_providers())), - api_(this, &grpc_xds_client_trace, bootstrap_->node(), - &bootstrap_->certificate_providers(), &symtab_) { + api_(this, &grpc_xds_client_trace, bootstrap_->node(), &symtab_, + std::move(user_agent_name), std::move(user_agent_version)), + engine_(std::move(engine)) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] creating xds client", this); } - // Calling grpc_init to ensure gRPC does not shut down until the XdsClient is - // destroyed. - grpc_init(); + GPR_ASSERT(bootstrap_ != nullptr); + if (bootstrap_->node() != nullptr) { + gpr_log(GPR_INFO, "[xds_client %p] xDS node ID: %s", this, + bootstrap_->node()->id().c_str()); + } } XdsClient::~XdsClient() { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] destroying xds client", this); } - grpc_channel_args_destroy(args_); - grpc_pollset_set_destroy(interested_parties_); - // Calling grpc_shutdown to ensure gRPC does not shut down until the XdsClient - // is destroyed. - grpc_shutdown(); } void XdsClient::Orphan() { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] shutting down xds client", this); } - { - MutexLock lock(g_mu); - if (g_xds_client == this) g_xds_client = nullptr; - } - { - MutexLock lock(&mu_); - shutting_down_ = true; - // Clear cache and any remaining watchers that may not have been cancelled. - authority_state_map_.clear(); - invalid_watchers_.clear(); + MutexLock lock(&mu_); + shutting_down_ = true; + // Clear cache and any remaining watchers that may not have been cancelled. + authority_state_map_.clear(); + invalid_watchers_.clear(); + // We may still be sending lingering queued load report data, so don't + // just clear the load reporting map, but we do want to clear the refs + // we're holding to the ChannelState objects, to make sure that + // everything shuts down properly. + for (auto& p : xds_load_report_server_map_) { + p.second.channel_state.reset(DEBUG_LOCATION, "XdsClient::Orphan()"); } } RefCountedPtr<XdsClient::ChannelState> XdsClient::GetOrCreateChannelStateLocked( - const XdsBootstrap::XdsServer& server) { - auto it = xds_server_channel_map_.find(server); + const XdsBootstrap::XdsServer& server, const char* reason) { + auto it = xds_server_channel_map_.find(&server); if (it != xds_server_channel_map_.end()) { - return it->second->Ref(DEBUG_LOCATION, "Authority"); + return it->second->Ref(DEBUG_LOCATION, reason); } // Channel not found, so create a new one. auto channel_state = MakeRefCounted<ChannelState>( WeakRef(DEBUG_LOCATION, "ChannelState"), server); - xds_server_channel_map_[server] = channel_state.get(); + xds_server_channel_map_[&server] = channel_state.get(); return channel_state; } @@ -1855,17 +1554,16 @@ void XdsClient::WatchResource(const XdsResourceType* type, invalid_watchers_[w] = watcher; } work_serializer_.Run( - // TODO(yashykt): When we move to C++14, capture watcher using - // std::move() - [watcher, status]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) { - watcher->OnError(status); - }, + [watcher = std::move(watcher), status = std::move(status)]() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) { + watcher->OnError(status); + }, DEBUG_LOCATION); }; auto resource_name = ParseXdsResourceName(name, type); if (!resource_name.ok()) { - fail(absl::UnavailableError(absl::StrFormat( - "Unable to parse resource name for listener %s", name))); + fail(absl::UnavailableError( + absl::StrCat("Unable to parse resource name ", name))); return; } // Find server to use. @@ -1879,11 +1577,17 @@ void XdsClient::WatchResource(const XdsResourceType* type, "\" not present in bootstrap config"))); return; } - if (!authority->xds_servers.empty()) { - xds_server = &authority->xds_servers[0]; - } + xds_server = authority->server(); } if (xds_server == nullptr) xds_server = &bootstrap_->server(); + // Canonify the xDS server instance, so that we make sure we're using + // the same instance as will be used in AddClusterDropStats() and + // AddClusterLocalityStats(). This may yield a different result than + // the logic above if the same server is listed both in the authority + // and as the top-level server. + // TODO(roth): This is really ugly -- need to find a better way to + // index the xDS server than by address here. + xds_server = bootstrap_->FindXdsServer(*xds_server); { MutexLock lock(&mu_); MaybeRegisterResourceTypeLocked(type); @@ -1907,12 +1611,60 @@ void XdsClient::WatchResource(const XdsResourceType* type, delete value; }, DEBUG_LOCATION); + } else if (resource_state.meta.client_status == + XdsApi::ResourceMetadata::DOES_NOT_EXIST) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, + "[xds_client %p] reporting cached does-not-exist for %s", this, + std::string(name).c_str()); + } + work_serializer_.Schedule( + [watcher]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) { + watcher->OnResourceDoesNotExist(); + }, + DEBUG_LOCATION); + } else if (resource_state.meta.client_status == + XdsApi::ResourceMetadata::NACKED) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log( + GPR_INFO, + "[xds_client %p] reporting cached validation failure for %s: %s", + this, std::string(name).c_str(), + resource_state.meta.failed_details.c_str()); + } + std::string details = resource_state.meta.failed_details; + const auto* node = bootstrap_->node(); + if (node != nullptr) { + absl::StrAppend(&details, " (node ID:", bootstrap_->node()->id(), ")"); + } + work_serializer_.Schedule( + [watcher, details = std::move(details)]() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) { + watcher->OnError(absl::UnavailableError( + absl::StrCat("invalid resource: ", details))); + }, + DEBUG_LOCATION); } // If the authority doesn't yet have a channel, set it, creating it if // needed. if (authority_state.channel_state == nullptr) { authority_state.channel_state = - GetOrCreateChannelStateLocked(*xds_server); + GetOrCreateChannelStateLocked(*xds_server, "start watch"); + } + absl::Status channel_status = authority_state.channel_state->status(); + if (!channel_status.ok()) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, + "[xds_client %p] returning cached channel error for %s: %s", + this, std::string(name).c_str(), + channel_status.ToString().c_str()); + } + work_serializer_.Schedule( + [watcher = std::move(watcher), status = std::move(channel_status)]() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) mutable { + watcher->OnError(std::move(status)); + }, + DEBUG_LOCATION); } authority_state.channel_state->SubscribeLocked(type, *resource_name); } @@ -1925,11 +1677,9 @@ void XdsClient::CancelResourceWatch(const XdsResourceType* type, bool delay_unsubscription) { auto resource_name = ParseXdsResourceName(name, type); MutexLock lock(&mu_); - if (!resource_name.ok()) { - invalid_watchers_.erase(watcher); - return; - } - if (shutting_down_) return; + // We cannot be sure whether the watcher is in invalid_watchers_ or in + // authority_state_map_, so we check both, just to be safe. + invalid_watchers_.erase(watcher); // Find authority. if (!resource_name.ok()) return; auto authority_it = authority_state_map_.find(resource_name->authority); @@ -1947,6 +1697,13 @@ void XdsClient::CancelResourceWatch(const XdsResourceType* type, resource_state.watchers.erase(watcher); // Clean up empty map entries, if any. if (resource_state.watchers.empty()) { + if (resource_state.ignored_deletion) { + gpr_log(GPR_INFO, + "[xds_client %p] unsubscribing from a resource for which we " + "previously ignored a deletion: type %s name %s", + this, std::string(type->type_url()).c_str(), + std::string(name).c_str()); + } authority_state.channel_state->UnsubscribeLocked(type, *resource_name, delay_unsubscription); type_map.erase(resource_it); @@ -1967,23 +1724,21 @@ void XdsClient::MaybeRegisterResourceTypeLocked( return; } resource_types_.emplace(resource_type->type_url(), resource_type); - v2_resource_types_.emplace(resource_type->v2_type_url(), resource_type); - resource_type->InitUpbSymtab(symtab_.ptr()); + resource_type->InitUpbSymtab(this, symtab_.ptr()); } const XdsResourceType* XdsClient::GetResourceTypeLocked( absl::string_view resource_type) { auto it = resource_types_.find(resource_type); if (it != resource_types_.end()) return it->second; - auto it2 = v2_resource_types_.find(resource_type); - if (it2 != v2_resource_types_.end()) return it2->second; return nullptr; } absl::StatusOr<XdsClient::XdsResourceName> XdsClient::ParseXdsResourceName( absl::string_view name, const XdsResourceType* type) { // Old-style names use the empty string for authority. - // authority is prefixed with "old:" to indicate that it's an old-style name. + // authority is prefixed with "old:" to indicate that it's an old-style + // name. if (!xds_federation_enabled_ || !absl::StartsWith(name, "xdstp:")) { return XdsResourceName{"old:", {std::string(name), {}}}; } @@ -1993,7 +1748,7 @@ absl::StatusOr<XdsClient::XdsResourceName> XdsClient::ParseXdsResourceName( // Split the resource type off of the path to get the id. std::pair<absl::string_view, absl::string_view> path_parts = absl::StrSplit( absl::StripPrefix(uri->path(), "/"), absl::MaxSplits('/', 1)); - if (!type->IsType(path_parts.first, nullptr)) { + if (type->type_url() != path_parts.first) { return absl::InvalidArgumentError( "xdstp URI path must indicate valid xDS resource type"); } @@ -2025,40 +1780,45 @@ std::string XdsClient::ConstructFullXdsResourceName( RefCountedPtr<XdsClusterDropStats> XdsClient::AddClusterDropStats( const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name, absl::string_view eds_service_name) { - if (!bootstrap_->XdsServerExists(xds_server)) return nullptr; + const auto* server = bootstrap_->FindXdsServer(xds_server); + if (server == nullptr) return nullptr; auto key = std::make_pair(std::string(cluster_name), std::string(eds_service_name)); - MutexLock lock(&mu_); - // We jump through some hoops here to make sure that the const - // XdsBootstrap::XdsServer& and absl::string_views - // stored in the XdsClusterDropStats object point to the - // XdsBootstrap::XdsServer and strings - // in the load_report_map_ key, so that they have the same lifetime. - auto server_it = - xds_load_report_server_map_.emplace(xds_server, LoadReportServer()).first; - if (server_it->second.channel_state == nullptr) { - server_it->second.channel_state = GetOrCreateChannelStateLocked(xds_server); - } - auto load_report_it = server_it->second.load_report_map - .emplace(std::move(key), LoadReportState()) - .first; - LoadReportState& load_report_state = load_report_it->second; RefCountedPtr<XdsClusterDropStats> cluster_drop_stats; - if (load_report_state.drop_stats != nullptr) { - cluster_drop_stats = load_report_state.drop_stats->RefIfNonZero(); - } - if (cluster_drop_stats == nullptr) { + { + MutexLock lock(&mu_); + // We jump through some hoops here to make sure that the const + // XdsBootstrap::XdsServer& and absl::string_views + // stored in the XdsClusterDropStats object point to the + // XdsBootstrap::XdsServer and strings + // in the load_report_map_ key, so that they have the same lifetime. + auto server_it = + xds_load_report_server_map_.emplace(server, LoadReportServer()).first; + if (server_it->second.channel_state == nullptr) { + server_it->second.channel_state = GetOrCreateChannelStateLocked( + *server, "load report map (drop stats)"); + } + auto load_report_it = server_it->second.load_report_map + .emplace(std::move(key), LoadReportState()) + .first; + LoadReportState& load_report_state = load_report_it->second; if (load_report_state.drop_stats != nullptr) { - load_report_state.deleted_drop_stats += - load_report_state.drop_stats->GetSnapshotAndReset(); + cluster_drop_stats = load_report_state.drop_stats->RefIfNonZero(); + } + if (cluster_drop_stats == nullptr) { + if (load_report_state.drop_stats != nullptr) { + load_report_state.deleted_drop_stats += + load_report_state.drop_stats->GetSnapshotAndReset(); + } + cluster_drop_stats = MakeRefCounted<XdsClusterDropStats>( + Ref(DEBUG_LOCATION, "DropStats"), *server, + load_report_it->first.first /*cluster_name*/, + load_report_it->first.second /*eds_service_name*/); + load_report_state.drop_stats = cluster_drop_stats.get(); } - cluster_drop_stats = MakeRefCounted<XdsClusterDropStats>( - Ref(DEBUG_LOCATION, "DropStats"), server_it->first, - load_report_it->first.first /*cluster_name*/, - load_report_it->first.second /*eds_service_name*/); - load_report_state.drop_stats = cluster_drop_stats.get(); + server_it->second.channel_state->MaybeStartLrsCall(); } - server_it->second.channel_state->MaybeStartLrsCall(); + work_serializer_.DrainQueue(); return cluster_drop_stats; } @@ -2066,8 +1826,10 @@ void XdsClient::RemoveClusterDropStats( const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name, absl::string_view eds_service_name, XdsClusterDropStats* cluster_drop_stats) { + const auto* server = bootstrap_->FindXdsServer(xds_server); + if (server == nullptr) return; MutexLock lock(&mu_); - auto server_it = xds_load_report_server_map_.find(xds_server); + auto server_it = xds_load_report_server_map_.find(server); if (server_it == xds_load_report_server_map_.end()) return; auto load_report_it = server_it->second.load_report_map.find( std::make_pair(std::string(cluster_name), std::string(eds_service_name))); @@ -2086,42 +1848,48 @@ RefCountedPtr<XdsClusterLocalityStats> XdsClient::AddClusterLocalityStats( const XdsBootstrap::XdsServer& xds_server, absl::string_view cluster_name, absl::string_view eds_service_name, RefCountedPtr<XdsLocalityName> locality) { - if (!bootstrap_->XdsServerExists(xds_server)) return nullptr; + const auto* server = bootstrap_->FindXdsServer(xds_server); + if (server == nullptr) return nullptr; auto key = std::make_pair(std::string(cluster_name), std::string(eds_service_name)); - MutexLock lock(&mu_); - // We jump through some hoops here to make sure that the const - // XdsBootstrap::XdsServer& and absl::string_views - // stored in the XdsClusterDropStats object point to the - // XdsBootstrap::XdsServer and strings - // in the load_report_map_ key, so that they have the same lifetime. - auto server_it = - xds_load_report_server_map_.emplace(xds_server, LoadReportServer()).first; - if (server_it->second.channel_state == nullptr) { - server_it->second.channel_state = GetOrCreateChannelStateLocked(xds_server); - } - auto load_report_it = server_it->second.load_report_map - .emplace(std::move(key), LoadReportState()) - .first; - LoadReportState& load_report_state = load_report_it->second; - LoadReportState::LocalityState& locality_state = - load_report_state.locality_stats[locality]; RefCountedPtr<XdsClusterLocalityStats> cluster_locality_stats; - if (locality_state.locality_stats != nullptr) { - cluster_locality_stats = locality_state.locality_stats->RefIfNonZero(); - } - if (cluster_locality_stats == nullptr) { + { + MutexLock lock(&mu_); + // We jump through some hoops here to make sure that the const + // XdsBootstrap::XdsServer& and absl::string_views + // stored in the XdsClusterDropStats object point to the + // XdsBootstrap::XdsServer and strings + // in the load_report_map_ key, so that they have the same lifetime. + auto server_it = + xds_load_report_server_map_.emplace(server, LoadReportServer()).first; + if (server_it->second.channel_state == nullptr) { + server_it->second.channel_state = GetOrCreateChannelStateLocked( + *server, "load report map (locality stats)"); + } + auto load_report_it = server_it->second.load_report_map + .emplace(std::move(key), LoadReportState()) + .first; + LoadReportState& load_report_state = load_report_it->second; + LoadReportState::LocalityState& locality_state = + load_report_state.locality_stats[locality]; if (locality_state.locality_stats != nullptr) { - locality_state.deleted_locality_stats += - locality_state.locality_stats->GetSnapshotAndReset(); + cluster_locality_stats = locality_state.locality_stats->RefIfNonZero(); } - cluster_locality_stats = MakeRefCounted<XdsClusterLocalityStats>( - Ref(DEBUG_LOCATION, "LocalityStats"), server_it->first, - load_report_it->first.first /*cluster_name*/, - load_report_it->first.second /*eds_service_name*/, std::move(locality)); - locality_state.locality_stats = cluster_locality_stats.get(); + if (cluster_locality_stats == nullptr) { + if (locality_state.locality_stats != nullptr) { + locality_state.deleted_locality_stats += + locality_state.locality_stats->GetSnapshotAndReset(); + } + cluster_locality_stats = MakeRefCounted<XdsClusterLocalityStats>( + Ref(DEBUG_LOCATION, "LocalityStats"), *server, + load_report_it->first.first /*cluster_name*/, + load_report_it->first.second /*eds_service_name*/, + std::move(locality)); + locality_state.locality_stats = cluster_locality_stats.get(); + } + server_it->second.channel_state->MaybeStartLrsCall(); } - server_it->second.channel_state->MaybeStartLrsCall(); + work_serializer_.DrainQueue(); return cluster_locality_stats; } @@ -2130,8 +1898,10 @@ void XdsClient::RemoveClusterLocalityStats( absl::string_view eds_service_name, const RefCountedPtr<XdsLocalityName>& locality, XdsClusterLocalityStats* cluster_locality_stats) { + const auto* server = bootstrap_->FindXdsServer(xds_server); + if (server == nullptr) return; MutexLock lock(&mu_); - auto server_it = xds_load_report_server_map_.find(xds_server); + auto server_it = xds_load_report_server_map_.find(server); if (server_it == xds_load_report_server_map_.end()) return; auto load_report_it = server_it->second.load_report_map.find( std::make_pair(std::string(cluster_name), std::string(eds_service_name))); @@ -2152,36 +1922,8 @@ void XdsClient::RemoveClusterLocalityStats( void XdsClient::ResetBackoff() { MutexLock lock(&mu_); for (auto& p : xds_server_channel_map_) { - grpc_channel_reset_connect_backoff(p.second->channel()); - } -} - -void XdsClient::NotifyOnErrorLocked(absl::Status status) { - const auto* node = bootstrap_->node(); - if (node != nullptr) { - status = absl::Status( - status.code(), absl::StrCat(status.message(), - " (node ID:", bootstrap_->node()->id, ")")); - } - std::set<RefCountedPtr<ResourceWatcherInterface>> watchers; - for (const auto& a : authority_state_map_) { // authority - for (const auto& t : a.second.resource_map) { // type - for (const auto& r : t.second) { // resource id - for (const auto& w : r.second.watchers) { // watchers - watchers.insert(w.second); - } - } - } + p.second->ResetBackoff(); } - work_serializer_.Schedule( - // TODO(yashykt): When we move to C++14, capture watchers using - // std::move() - [watchers, status]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(work_serializer_) { - for (const auto& watcher : watchers) { - watcher->OnError(status); - } - }, - DEBUG_LOCATION); } void XdsClient::NotifyWatchersOnErrorLocked( @@ -2191,15 +1933,16 @@ void XdsClient::NotifyWatchersOnErrorLocked( const auto* node = bootstrap_->node(); if (node != nullptr) { status = absl::Status( - status.code(), absl::StrCat(status.message(), - " (node ID:", bootstrap_->node()->id, ")")); + status.code(), + absl::StrCat(status.message(), " (node ID:", node->id(), ")")); } work_serializer_.Schedule( - [watchers, status]() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) { - for (const auto& p : watchers) { - p.first->OnError(status); - } - }, + [watchers, status = std::move(status)]() + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) { + for (const auto& p : watchers) { + p.first->OnError(status); + } + }, DEBUG_LOCATION); } @@ -2222,7 +1965,7 @@ XdsApi::ClusterLoadReportMap XdsClient::BuildLoadReportSnapshotLocked( gpr_log(GPR_INFO, "[xds_client %p] start building load report", this); } XdsApi::ClusterLoadReportMap snapshot_map; - auto server_it = xds_load_report_server_map_.find(xds_server); + auto server_it = xds_load_report_server_map_.find(&xds_server); if (server_it == xds_load_report_server_map_.end()) return snapshot_map; auto& load_report_map = server_it->second.load_report_map; for (auto load_report_it = load_report_map.begin(); @@ -2281,7 +2024,7 @@ XdsApi::ClusterLoadReportMap XdsClient::BuildLoadReportSnapshotLocked( } } // Compute load report interval. - const Timestamp now = ExecCtx::Get()->Now(); + const Timestamp now = Timestamp::Now(); snapshot.load_report_interval = now - load_report.last_report_time; load_report.last_report_time = now; // Record snapshot. @@ -2321,192 +2064,4 @@ std::string XdsClient::DumpClientConfigBinary() { return api_.AssembleClientConfig(resource_type_metadata_map); } -// -// accessors for global state -// - -void XdsClientGlobalInit() { - g_mu = new Mutex; - XdsHttpFilterRegistry::Init(); - XdsClusterSpecifierPluginRegistry::Init(); -} - -// TODO(roth): Find a better way to clear the fallback config that does -// not require using ABSL_NO_THREAD_SAFETY_ANALYSIS. -void XdsClientGlobalShutdown() ABSL_NO_THREAD_SAFETY_ANALYSIS { - gpr_free(g_fallback_bootstrap_config); - g_fallback_bootstrap_config = nullptr; - delete g_mu; - g_mu = nullptr; - XdsHttpFilterRegistry::Shutdown(); - XdsClusterSpecifierPluginRegistry::Shutdown(); -} - -namespace { - -std::string GetBootstrapContents(const char* fallback_config, - grpc_error_handle* error) { - // First, try GRPC_XDS_BOOTSTRAP env var. - UniquePtr<char> path(gpr_getenv("GRPC_XDS_BOOTSTRAP")); - if (path != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log(GPR_INFO, - "Got bootstrap file location from GRPC_XDS_BOOTSTRAP " - "environment variable: %s", - path.get()); - } - grpc_slice contents; - *error = - grpc_load_file(path.get(), /*add_null_terminator=*/true, &contents); - if (*error != GRPC_ERROR_NONE) return ""; - std::string contents_str(StringViewFromSlice(contents)); - grpc_slice_unref_internal(contents); - return contents_str; - } - // Next, try GRPC_XDS_BOOTSTRAP_CONFIG env var. - UniquePtr<char> env_config(gpr_getenv("GRPC_XDS_BOOTSTRAP_CONFIG")); - if (env_config != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log(GPR_INFO, - "Got bootstrap contents from GRPC_XDS_BOOTSTRAP_CONFIG " - "environment variable"); - } - return env_config.get(); - } - // Finally, try fallback config. - if (fallback_config != nullptr) { - if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log(GPR_INFO, "Got bootstrap contents from fallback config"); - } - return fallback_config; - } - // No bootstrap config found. - *error = GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Environment variables GRPC_XDS_BOOTSTRAP or GRPC_XDS_BOOTSTRAP_CONFIG " - "not defined"); - return ""; -} - -} // namespace - -RefCountedPtr<XdsClient> XdsClient::GetOrCreate(const grpc_channel_args* args, - grpc_error_handle* error) { - RefCountedPtr<XdsClient> xds_client; - // If getting bootstrap from channel args, create a local XdsClient - // instance for the channel or server instead of using the global instance. - const char* bootstrap_config = grpc_channel_args_find_string( - args, GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_BOOTSTRAP_CONFIG); - if (bootstrap_config != nullptr) { - std::unique_ptr<XdsBootstrap> bootstrap = - XdsBootstrap::Create(bootstrap_config, error); - if (*error == GRPC_ERROR_NONE) { - grpc_channel_args* xds_channel_args = - grpc_channel_args_find_pointer<grpc_channel_args>( - args, - GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS); - return MakeRefCounted<XdsClient>(std::move(bootstrap), xds_channel_args); - } - return nullptr; - } - // Otherwise, use the global instance. - { - MutexLock lock(g_mu); - if (g_xds_client != nullptr) { - auto xds_client = g_xds_client->RefIfNonZero(); - if (xds_client != nullptr) return xds_client; - } - // Find bootstrap contents. - std::string bootstrap_contents = - GetBootstrapContents(g_fallback_bootstrap_config, error); - if (*error != GRPC_ERROR_NONE) return nullptr; - if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { - gpr_log(GPR_INFO, "xDS bootstrap contents: %s", - bootstrap_contents.c_str()); - } - // Parse bootstrap. - std::unique_ptr<XdsBootstrap> bootstrap = - XdsBootstrap::Create(bootstrap_contents, error); - if (*error != GRPC_ERROR_NONE) return nullptr; - // Instantiate XdsClient. - xds_client = - MakeRefCounted<XdsClient>(std::move(bootstrap), g_channel_args); - g_xds_client = xds_client.get(); - } - return xds_client; -} - -namespace internal { - -void SetXdsChannelArgsForTest(grpc_channel_args* args) { - MutexLock lock(g_mu); - g_channel_args = args; -} - -void UnsetGlobalXdsClientForTest() { - MutexLock lock(g_mu); - g_xds_client = nullptr; -} - -void SetXdsFallbackBootstrapConfig(const char* config) { - MutexLock lock(g_mu); - gpr_free(g_fallback_bootstrap_config); - g_fallback_bootstrap_config = gpr_strdup(config); -} - -} // namespace internal - -// -// embedding XdsClient in channel args -// - -#define GRPC_ARG_XDS_CLIENT "grpc.internal.xds_client" - -namespace { - -void* XdsClientArgCopy(void* p) { - XdsClient* xds_client = static_cast<XdsClient*>(p); - xds_client->Ref(DEBUG_LOCATION, "channel arg").release(); - return p; -} - -void XdsClientArgDestroy(void* p) { - XdsClient* xds_client = static_cast<XdsClient*>(p); - xds_client->Unref(DEBUG_LOCATION, "channel arg"); -} - -int XdsClientArgCmp(void* p, void* q) { return QsortCompare(p, q); } - -const grpc_arg_pointer_vtable kXdsClientArgVtable = { - XdsClientArgCopy, XdsClientArgDestroy, XdsClientArgCmp}; - -} // namespace - -grpc_arg XdsClient::MakeChannelArg() const { - return grpc_channel_arg_pointer_create(const_cast<char*>(GRPC_ARG_XDS_CLIENT), - const_cast<XdsClient*>(this), - &kXdsClientArgVtable); -} - -RefCountedPtr<XdsClient> XdsClient::GetFromChannelArgs( - const grpc_channel_args& args) { - XdsClient* xds_client = - grpc_channel_args_find_pointer<XdsClient>(&args, GRPC_ARG_XDS_CLIENT); - if (xds_client == nullptr) return nullptr; - return xds_client->Ref(DEBUG_LOCATION, "GetFromChannelArgs"); -} - } // namespace grpc_core - -// The returned bytes may contain NULL(0), so we can't use c-string. -grpc_slice grpc_dump_xds_configs() { - grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; - grpc_core::ExecCtx exec_ctx; - grpc_error_handle error = GRPC_ERROR_NONE; - auto xds_client = grpc_core::XdsClient::GetOrCreate(nullptr, &error); - if (error != GRPC_ERROR_NONE) { - // If we isn't using xDS, just return an empty string. - GRPC_ERROR_UNREF(error); - return grpc_empty_slice(); - } - return grpc_slice_from_cpp_string(xds_client->DumpClientConfigBinary()); -} diff --git a/grpc/src/core/ext/xds/xds_client.h b/grpc/src/core/ext/xds/xds_client.h index 5fe050cc..e8303d27 100644 --- a/grpc/src/core/ext/xds/xds_client.h +++ b/grpc/src/core/ext/xds/xds_client.h @@ -14,29 +14,39 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_CLIENT_H -#define GRPC_CORE_EXT_XDS_XDS_CLIENT_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_CLIENT_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_CLIENT_H #include <grpc/support/port_platform.h> +#include <map> +#include <memory> #include <set> +#include <string> +#include <utility> #include <vector> +#include "absl/base/thread_annotations.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/string_view.h" -#include "absl/types/optional.h" +#include "upb/reflection/def.hpp" + +#include <grpc/event_engine/event_engine.h> #include "src/core/ext/xds/xds_api.h" #include "src/core/ext/xds/xds_bootstrap.h" #include "src/core/ext/xds/xds_client_stats.h" #include "src/core/ext/xds/xds_resource_type.h" -#include "src/core/lib/channel/channelz.h" +#include "src/core/ext/xds/xds_transport.h" +#include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/dual_ref_counted.h" -#include "src/core/lib/gprpp/memory.h" #include "src/core/lib/gprpp/orphanable.h" #include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/sync.h" -#include "src/core/lib/iomgr/work_serializer.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/gprpp/work_serializer.h" #include "src/core/lib/uri/uri_parser.h" namespace grpc_core { @@ -61,29 +71,22 @@ class XdsClient : public DualRefCounted<XdsClient> { ABSL_EXCLUSIVE_LOCKS_REQUIRED(&work_serializer_) = 0; }; - // Factory function to get or create the global XdsClient instance. - // If *error is not GRPC_ERROR_NONE upon return, then there was - // an error initializing the client. - static RefCountedPtr<XdsClient> GetOrCreate(const grpc_channel_args* args, - grpc_error_handle* error); - - // Most callers should not instantiate directly. Use GetOrCreate() instead. - XdsClient(std::unique_ptr<XdsBootstrap> bootstrap, - const grpc_channel_args* args); + XdsClient( + std::unique_ptr<XdsBootstrap> bootstrap, + OrphanablePtr<XdsTransportFactory> transport_factory, + std::shared_ptr<grpc_event_engine::experimental::EventEngine> engine, + std::string user_agent_name, std::string user_agent_version, + Duration resource_request_timeout = Duration::Seconds(15)); ~XdsClient() override; const XdsBootstrap& bootstrap() const { - // bootstrap_ is guaranteed to be non-null since XdsClient::GetOrCreate() - // would return a null object if bootstrap_ was null. - return *bootstrap_; + return *bootstrap_; // ctor asserts that it is non-null } - CertificateProviderStore& certificate_provider_store() { - return *certificate_provider_store_; + XdsTransportFactory* transport_factory() const { + return transport_factory_.get(); } - grpc_pollset_set* interested_parties() const { return interested_parties_; } - void Orphan() override; // Start and cancel watch for a resource. @@ -145,10 +148,9 @@ class XdsClient : public DualRefCounted<XdsClient> { // implementation. std::string DumpClientConfigBinary(); - // Helpers for encoding the XdsClient object in channel args. - grpc_arg MakeChannelArg() const; - static RefCountedPtr<XdsClient> GetFromChannelArgs( - const grpc_channel_args& args); + grpc_event_engine::experimental::EventEngine* engine() { + return engine_.get(); + } private: struct XdsResourceKey { @@ -183,20 +185,18 @@ class XdsClient : public DualRefCounted<XdsClient> { void Orphan() override; - grpc_channel* channel() const { return channel_; } XdsClient* xds_client() const { return xds_client_.get(); } AdsCallState* ads_calld() const; LrsCallState* lrs_calld() const; + void ResetBackoff(); + void MaybeStartLrsCall(); void StopLrsCallLocked() ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); - bool HasAdsCall() const; - bool HasActiveAdsCall() const; - - void StartConnectivityWatchLocked() - ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); - void CancelConnectivityWatchLocked(); + // Returns non-OK if there has been an error since the last time the + // ADS stream saw a response. + const absl::Status& status() const { return status_; } void SubscribeLocked(const XdsResourceType* type, const XdsResourceName& name) @@ -207,17 +207,21 @@ class XdsClient : public DualRefCounted<XdsClient> { ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); private: - class StateWatcher; + void OnConnectivityFailure(absl::Status status); + + // Enqueues error notifications to watchers. Caller must drain + // XdsClient::work_serializer_ after releasing the lock. + void SetChannelStatusLocked(absl::Status status) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(&XdsClient::mu_); // The owning xds client. WeakRefCountedPtr<XdsClient> xds_client_; - const XdsBootstrap::XdsServer& server_; + const XdsBootstrap::XdsServer& server_; // Owned by bootstrap. + + OrphanablePtr<XdsTransportFactory::XdsTransport> transport_; - // The channel and its status. - grpc_channel* channel_; bool shutting_down_ = false; - StateWatcher* watcher_; // The retryable XDS calls. OrphanablePtr<RetryableCall<AdsCallState>> ads_calld_; @@ -226,6 +230,8 @@ class XdsClient : public DualRefCounted<XdsClient> { // Stores the most recent accepted resource version for each resource type. std::map<const XdsResourceType*, std::string /*version*/> resource_type_version_map_; + + absl::Status status_; }; struct ResourceState { @@ -234,6 +240,7 @@ class XdsClient : public DualRefCounted<XdsClient> { // The latest data seen for the resource. std::unique_ptr<XdsResourceType::ResourceData> resource; XdsApi::ResourceMetadata meta; + bool ignored_deletion = false; }; struct AuthorityState { @@ -253,7 +260,7 @@ class XdsClient : public DualRefCounted<XdsClient> { std::map<RefCountedPtr<XdsLocalityName>, LocalityState, XdsLocalityName::Less> locality_stats; - Timestamp last_report_time = ExecCtx::Get()->Now(); + Timestamp last_report_time = Timestamp::Now(); }; // Load report data. @@ -266,9 +273,6 @@ class XdsClient : public DualRefCounted<XdsClient> { LoadReportMap load_report_map; }; - // Sends an error notification to all watchers. - void NotifyOnErrorLocked(absl::Status status) - ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Sends an error notification to a specific set of watchers. void NotifyWatchersOnErrorLocked( const std::map<ResourceWatcherInterface*, @@ -297,34 +301,34 @@ class XdsClient : public DualRefCounted<XdsClient> { const std::set<std::string>& clusters) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); RefCountedPtr<ChannelState> GetOrCreateChannelStateLocked( - const XdsBootstrap::XdsServer& server) ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); + const XdsBootstrap::XdsServer& server, const char* reason) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(mu_); std::unique_ptr<XdsBootstrap> bootstrap_; - grpc_channel_args* args_; + OrphanablePtr<XdsTransportFactory> transport_factory_; const Duration request_timeout_; const bool xds_federation_enabled_; - grpc_pollset_set* interested_parties_; - OrphanablePtr<CertificateProviderStore> certificate_provider_store_; XdsApi api_; WorkSerializer work_serializer_; + std::shared_ptr<grpc_event_engine::experimental::EventEngine> engine_; Mutex mu_; // Stores resource type objects seen by type URL. std::map<absl::string_view /*resource_type*/, const XdsResourceType*> resource_types_ ABSL_GUARDED_BY(mu_); - std::map<absl::string_view /*v2_resource_type*/, const XdsResourceType*> - v2_resource_types_ ABSL_GUARDED_BY(mu_); upb::SymbolTable symtab_ ABSL_GUARDED_BY(mu_); - // Map of existing xDS server channels. - std::map<XdsBootstrap::XdsServer, ChannelState*> xds_server_channel_map_ - ABSL_GUARDED_BY(mu_); + // Map of existing xDS server channels. + // Key is owned by the bootstrap config. + std::map<const XdsBootstrap::XdsServer*, ChannelState*> + xds_server_channel_map_ ABSL_GUARDED_BY(mu_); std::map<std::string /*authority*/, AuthorityState> authority_state_map_ ABSL_GUARDED_BY(mu_); - std::map<XdsBootstrap::XdsServer, LoadReportServer> + // Key is owned by the bootstrap config. + std::map<const XdsBootstrap::XdsServer*, LoadReportServer> xds_load_report_server_map_ ABSL_GUARDED_BY(mu_); // Stores started watchers whose resource name was not parsed successfully, @@ -335,14 +339,6 @@ class XdsClient : public DualRefCounted<XdsClient> { bool shutting_down_ ABSL_GUARDED_BY(mu_) = false; }; -namespace internal { -void SetXdsChannelArgsForTest(grpc_channel_args* args); -void UnsetGlobalXdsClientForTest(); -// Sets bootstrap config to be used when no env var is set. -// Does not take ownership of config. -void SetXdsFallbackBootstrapConfig(const char* config); -} // namespace internal - } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_CLIENT_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_CLIENT_H diff --git a/grpc/src/core/ext/xds/xds_client_grpc.cc b/grpc/src/core/ext/xds/xds_client_grpc.cc new file mode 100644 index 00000000..63ad3080 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_client_grpc.cc @@ -0,0 +1,235 @@ +// +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include <grpc/support/port_platform.h> + +#include "src/core/ext/xds/xds_client_grpc.h" + +#include <algorithm> +#include <memory> +#include <string> +#include <utility> + +#include "absl/base/thread_annotations.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" + +#include <grpc/grpc.h> +#include <grpc/slice.h> +#include <grpc/support/alloc.h> +#include <grpc/support/log.h> +#include <grpc/support/string_util.h> + +#include "src/core/ext/xds/xds_bootstrap.h" +#include "src/core/ext/xds/xds_bootstrap_grpc.h" +#include "src/core/ext/xds/xds_channel_args.h" +#include "src/core/ext/xds/xds_transport.h" +#include "src/core/ext/xds/xds_transport_grpc.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/event_engine/default_event_engine.h" +#include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/env.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/sync.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/iomgr/load_file.h" +#include "src/core/lib/slice/slice.h" +#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/transport/error_utils.h" + +namespace grpc_core { + +// If gRPC is built with -DGRPC_XDS_USER_AGENT_NAME_SUFFIX="...", that string +// will be appended to the user agent name reported to the xDS server. +#ifdef GRPC_XDS_USER_AGENT_NAME_SUFFIX +#define GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING \ + " " GRPC_XDS_USER_AGENT_NAME_SUFFIX +#else +#define GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING "" +#endif + +// If gRPC is built with -DGRPC_XDS_USER_AGENT_VERSION_SUFFIX="...", that string +// will be appended to the user agent version reported to the xDS server. +#ifdef GRPC_XDS_USER_AGENT_VERSION_SUFFIX +#define GRPC_XDS_USER_AGENT_VERSION_SUFFIX_STRING \ + " " GRPC_XDS_USER_AGENT_VERSION_SUFFIX +#else +#define GRPC_XDS_USER_AGENT_VERSION_SUFFIX_STRING "" +#endif + +// +// GrpcXdsClient +// + +namespace { + +Mutex* g_mu = new Mutex; +const grpc_channel_args* g_channel_args ABSL_GUARDED_BY(*g_mu) = nullptr; +GrpcXdsClient* g_xds_client ABSL_GUARDED_BY(*g_mu) = nullptr; +char* g_fallback_bootstrap_config ABSL_GUARDED_BY(*g_mu) = nullptr; + +} // namespace + +namespace { + +absl::StatusOr<std::string> GetBootstrapContents(const char* fallback_config) { + // First, try GRPC_XDS_BOOTSTRAP env var. + auto path = GetEnv("GRPC_XDS_BOOTSTRAP"); + if (path.has_value()) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, + "Got bootstrap file location from GRPC_XDS_BOOTSTRAP " + "environment variable: %s", + path->c_str()); + } + grpc_slice contents; + grpc_error_handle error = + grpc_load_file(path->c_str(), /*add_null_terminator=*/true, &contents); + if (!error.ok()) return grpc_error_to_absl_status(error); + std::string contents_str(StringViewFromSlice(contents)); + CSliceUnref(contents); + return contents_str; + } + // Next, try GRPC_XDS_BOOTSTRAP_CONFIG env var. + auto env_config = GetEnv("GRPC_XDS_BOOTSTRAP_CONFIG"); + if (env_config.has_value()) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, + "Got bootstrap contents from GRPC_XDS_BOOTSTRAP_CONFIG " + "environment variable"); + } + return std::move(*env_config); + } + // Finally, try fallback config. + if (fallback_config != nullptr) { + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, "Got bootstrap contents from fallback config"); + } + return fallback_config; + } + // No bootstrap config found. + return absl::FailedPreconditionError( + "Environment variables GRPC_XDS_BOOTSTRAP or GRPC_XDS_BOOTSTRAP_CONFIG " + "not defined"); +} + +} // namespace + +absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GrpcXdsClient::GetOrCreate( + const ChannelArgs& args, const char* reason) { + // If getting bootstrap from channel args, create a local XdsClient + // instance for the channel or server instead of using the global instance. + absl::optional<absl::string_view> bootstrap_config = args.GetString( + GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_BOOTSTRAP_CONFIG); + if (bootstrap_config.has_value()) { + auto bootstrap = GrpcXdsBootstrap::Create(*bootstrap_config); + if (!bootstrap.ok()) return bootstrap.status(); + grpc_channel_args* xds_channel_args = args.GetPointer<grpc_channel_args>( + GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS); + return MakeRefCounted<GrpcXdsClient>(std::move(*bootstrap), + ChannelArgs::FromC(xds_channel_args)); + } + // Otherwise, use the global instance. + MutexLock lock(g_mu); + if (g_xds_client != nullptr) { + auto xds_client = g_xds_client->RefIfNonZero(DEBUG_LOCATION, reason); + if (xds_client != nullptr) return xds_client; + } + // Find bootstrap contents. + auto bootstrap_contents = GetBootstrapContents(g_fallback_bootstrap_config); + if (!bootstrap_contents.ok()) return bootstrap_contents.status(); + if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { + gpr_log(GPR_INFO, "xDS bootstrap contents: %s", + bootstrap_contents->c_str()); + } + // Parse bootstrap. + auto bootstrap = GrpcXdsBootstrap::Create(*bootstrap_contents); + if (!bootstrap.ok()) return bootstrap.status(); + // Instantiate XdsClient. + auto xds_client = MakeRefCounted<GrpcXdsClient>( + std::move(*bootstrap), ChannelArgs::FromC(g_channel_args)); + g_xds_client = xds_client.get(); + return xds_client; +} + +GrpcXdsClient::GrpcXdsClient(std::unique_ptr<GrpcXdsBootstrap> bootstrap, + const ChannelArgs& args) + : XdsClient( + std::move(bootstrap), MakeOrphanable<GrpcXdsTransportFactory>(args), + grpc_event_engine::experimental::GetDefaultEventEngine(), + absl::StrCat("gRPC C-core ", GPR_PLATFORM_STRING, + GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING), + absl::StrCat("C-core ", grpc_version_string(), + GRPC_XDS_USER_AGENT_NAME_SUFFIX_STRING, + GRPC_XDS_USER_AGENT_VERSION_SUFFIX_STRING), + std::max(Duration::Zero(), + args.GetDurationFromIntMillis( + GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS) + .value_or(Duration::Seconds(15)))), + certificate_provider_store_(MakeOrphanable<CertificateProviderStore>( + static_cast<const GrpcXdsBootstrap&>(this->bootstrap()) + .certificate_providers())) {} + +GrpcXdsClient::~GrpcXdsClient() { + MutexLock lock(g_mu); + if (g_xds_client == this) g_xds_client = nullptr; +} + +grpc_pollset_set* GrpcXdsClient::interested_parties() const { + return reinterpret_cast<GrpcXdsTransportFactory*>(transport_factory()) + ->interested_parties(); +} + +namespace internal { + +void SetXdsChannelArgsForTest(grpc_channel_args* args) { + MutexLock lock(g_mu); + g_channel_args = args; +} + +void UnsetGlobalXdsClientForTest() { + MutexLock lock(g_mu); + g_xds_client = nullptr; +} + +void SetXdsFallbackBootstrapConfig(const char* config) { + MutexLock lock(g_mu); + gpr_free(g_fallback_bootstrap_config); + g_fallback_bootstrap_config = gpr_strdup(config); +} + +} // namespace internal + +} // namespace grpc_core + +// The returned bytes may contain NULL(0), so we can't use c-string. +grpc_slice grpc_dump_xds_configs(void) { + grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; + grpc_core::ExecCtx exec_ctx; + auto xds_client = grpc_core::GrpcXdsClient::GetOrCreate( + grpc_core::ChannelArgs(), "grpc_dump_xds_configs()"); + if (!xds_client.ok()) { + // If we aren't using xDS, just return an empty string. + return grpc_empty_slice(); + } + return grpc_slice_from_cpp_string((*xds_client)->DumpClientConfigBinary()); +} diff --git a/grpc/src/core/ext/xds/xds_client_grpc.h b/grpc/src/core/ext/xds/xds_client_grpc.h new file mode 100644 index 00000000..7c275b0a --- /dev/null +++ b/grpc/src/core/ext/xds/xds_client_grpc.h @@ -0,0 +1,79 @@ +// +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_CLIENT_GRPC_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_CLIENT_GRPC_H + +#include <grpc/support/port_platform.h> + +#include <memory> + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" + +#include <grpc/grpc.h> + +#include "src/core/ext/xds/certificate_provider_store.h" +#include "src/core/ext/xds/xds_bootstrap_grpc.h" +#include "src/core/ext/xds/xds_client.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gpr/useful.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/iomgr_fwd.h" + +namespace grpc_core { + +class GrpcXdsClient : public XdsClient { + public: + // Factory function to get or create the global XdsClient instance. + static absl::StatusOr<RefCountedPtr<GrpcXdsClient>> GetOrCreate( + const ChannelArgs& args, const char* reason); + + // Do not instantiate directly -- use GetOrCreate() instead. + GrpcXdsClient(std::unique_ptr<GrpcXdsBootstrap> bootstrap, + const ChannelArgs& args); + ~GrpcXdsClient() override; + + // Helpers for encoding the XdsClient object in channel args. + static absl::string_view ChannelArgName() { + return "grpc.internal.xds_client"; + } + static int ChannelArgsCompare(const XdsClient* a, const XdsClient* b) { + return QsortCompare(a, b); + } + + grpc_pollset_set* interested_parties() const; + + CertificateProviderStore& certificate_provider_store() const { + return *certificate_provider_store_; + } + + private: + OrphanablePtr<CertificateProviderStore> certificate_provider_store_; +}; + +namespace internal { +void SetXdsChannelArgsForTest(grpc_channel_args* args); +void UnsetGlobalXdsClientForTest(); +// Sets bootstrap config to be used when no env var is set. +// Does not take ownership of config. +void SetXdsFallbackBootstrapConfig(const char* config); +} // namespace internal + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_CLIENT_GRPC_H diff --git a/grpc/src/core/ext/xds/xds_client_stats.cc b/grpc/src/core/ext/xds/xds_client_stats.cc index 3cd26527..512d5996 100644 --- a/grpc/src/core/ext/xds/xds_client_stats.cc +++ b/grpc/src/core/ext/xds/xds_client_stats.cc @@ -1,31 +1,30 @@ -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ +// +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// #include <grpc/support/port_platform.h> #include "src/core/ext/xds/xds_client_stats.h" -#include <string.h> - -#include <grpc/support/atm.h> -#include <grpc/support/string_util.h> +#include <grpc/support/log.h> #include "src/core/ext/xds/xds_client.h" +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/debug_location.h" namespace grpc_core { @@ -54,7 +53,7 @@ XdsClusterDropStats::XdsClusterDropStats( eds_service_name_(eds_service_name) { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] created drop stats %p for {%s, %s, %s}", - xds_client_.get(), this, lrs_server_.server_uri.c_str(), + xds_client_.get(), this, lrs_server_.server_uri().c_str(), std::string(cluster_name_).c_str(), std::string(eds_service_name_).c_str()); } @@ -64,7 +63,7 @@ XdsClusterDropStats::~XdsClusterDropStats() { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] destroying drop stats %p for {%s, %s, %s}", - xds_client_.get(), this, lrs_server_.server_uri.c_str(), + xds_client_.get(), this, lrs_server_.server_uri().c_str(), std::string(cluster_name_).c_str(), std::string(eds_service_name_).c_str()); } @@ -109,7 +108,7 @@ XdsClusterLocalityStats::XdsClusterLocalityStats( if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] created locality stats %p for {%s, %s, %s, %s}", - xds_client_.get(), this, lrs_server_.server_uri.c_str(), + xds_client_.get(), this, lrs_server_.server_uri().c_str(), std::string(cluster_name_).c_str(), std::string(eds_service_name_).c_str(), name_->AsHumanReadableString().c_str()); @@ -120,7 +119,7 @@ XdsClusterLocalityStats::~XdsClusterLocalityStats() { if (GRPC_TRACE_FLAG_ENABLED(grpc_xds_client_trace)) { gpr_log(GPR_INFO, "[xds_client %p] destroying locality stats %p for {%s, %s, %s, %s}", - xds_client_.get(), this, lrs_server_.server_uri.c_str(), + xds_client_.get(), this, lrs_server_.server_uri().c_str(), std::string(cluster_name_).c_str(), std::string(eds_service_name_).c_str(), name_->AsHumanReadableString().c_str()); @@ -132,29 +131,43 @@ XdsClusterLocalityStats::~XdsClusterLocalityStats() { XdsClusterLocalityStats::Snapshot XdsClusterLocalityStats::GetSnapshotAndReset() { - Snapshot snapshot = { - GetAndResetCounter(&total_successful_requests_), - // Don't reset total_requests_in_progress because it's - // not related to a single reporting interval. - total_requests_in_progress_.load(std::memory_order_relaxed), - GetAndResetCounter(&total_error_requests_), - GetAndResetCounter(&total_issued_requests_), - {}}; - MutexLock lock(&backend_metrics_mu_); - snapshot.backend_metrics = std::move(backend_metrics_); + Snapshot snapshot; + for (auto& percpu_stats : stats_) { + Snapshot percpu_snapshot = { + GetAndResetCounter(&percpu_stats.total_successful_requests), + // Don't reset total_requests_in_progress because it's + // not related to a single reporting interval. + percpu_stats.total_requests_in_progress.load(std::memory_order_relaxed), + GetAndResetCounter(&percpu_stats.total_error_requests), + GetAndResetCounter(&percpu_stats.total_issued_requests), + {}}; + { + MutexLock lock(&percpu_stats.backend_metrics_mu); + percpu_snapshot.backend_metrics = std::move(percpu_stats.backend_metrics); + } + snapshot += percpu_snapshot; + } return snapshot; } void XdsClusterLocalityStats::AddCallStarted() { - total_issued_requests_.fetch_add(1, std::memory_order_relaxed); - total_requests_in_progress_.fetch_add(1, std::memory_order_relaxed); + Stats& stats = stats_.this_cpu(); + stats.total_issued_requests.fetch_add(1, std::memory_order_relaxed); + stats.total_requests_in_progress.fetch_add(1, std::memory_order_relaxed); } -void XdsClusterLocalityStats::AddCallFinished(bool fail) { +void XdsClusterLocalityStats::AddCallFinished( + const std::map<absl::string_view, double>* named_metrics, bool fail) { + Stats& stats = stats_.this_cpu(); std::atomic<uint64_t>& to_increment = - fail ? total_error_requests_ : total_successful_requests_; + fail ? stats.total_error_requests : stats.total_successful_requests; to_increment.fetch_add(1, std::memory_order_relaxed); - total_requests_in_progress_.fetch_add(-1, std::memory_order_acq_rel); + stats.total_requests_in_progress.fetch_add(-1, std::memory_order_acq_rel); + if (named_metrics == nullptr) return; + MutexLock lock(&stats.backend_metrics_mu); + for (const auto& m : *named_metrics) { + stats.backend_metrics[std::string(m.first)] += BackendMetric{1, m.second}; + } } } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_client_stats.h b/grpc/src/core/ext/xds/xds_client_stats.h index 9ed5cf4a..3f3a776f 100644 --- a/grpc/src/core/ext/xds/xds_client_stats.h +++ b/grpc/src/core/ext/xds/xds_client_stats.h @@ -1,40 +1,43 @@ -/* - * - * Copyright 2018 gRPC authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -#ifndef GRPC_CORE_EXT_XDS_XDS_CLIENT_STATS_H -#define GRPC_CORE_EXT_XDS_XDS_CLIENT_STATS_H +// +// +// Copyright 2018 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// + +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_CLIENT_STATS_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_CLIENT_STATS_H #include <grpc/support/port_platform.h> #include <atomic> +#include <cstdint> +#include <initializer_list> #include <map> #include <string> +#include <utility> -#include "absl/strings/str_cat.h" +#include "absl/base/thread_annotations.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" #include "src/core/ext/xds/xds_bootstrap.h" #include "src/core/lib/gpr/useful.h" -#include "src/core/lib/gprpp/memory.h" +#include "src/core/lib/gprpp/per_cpu.h" #include "src/core/lib/gprpp/ref_counted.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" #include "src/core/lib/gprpp/sync.h" -#include "src/core/lib/iomgr/exec_ctx.h" namespace grpc_core { @@ -157,8 +160,8 @@ class XdsClusterDropStats : public RefCounted<XdsClusterDropStats> { class XdsClusterLocalityStats : public RefCounted<XdsClusterLocalityStats> { public: struct BackendMetric { - uint64_t num_requests_finished_with_metric; - double total_metric_value; + uint64_t num_requests_finished_with_metric = 0; + double total_metric_value = 0; BackendMetric& operator+=(const BackendMetric& other) { num_requests_finished_with_metric += @@ -173,10 +176,10 @@ class XdsClusterLocalityStats : public RefCounted<XdsClusterLocalityStats> { }; struct Snapshot { - uint64_t total_successful_requests; - uint64_t total_requests_in_progress; - uint64_t total_error_requests; - uint64_t total_issued_requests; + uint64_t total_successful_requests = 0; + uint64_t total_requests_in_progress = 0; + uint64_t total_error_requests = 0; + uint64_t total_issued_requests = 0; std::map<std::string, BackendMetric> backend_metrics; Snapshot& operator+=(const Snapshot& other) { @@ -213,29 +216,32 @@ class XdsClusterLocalityStats : public RefCounted<XdsClusterLocalityStats> { Snapshot GetSnapshotAndReset(); void AddCallStarted(); - void AddCallFinished(bool fail = false); + void AddCallFinished(const std::map<absl::string_view, double>* named_metrics, + bool fail = false); private: + struct Stats { + std::atomic<uint64_t> total_successful_requests{0}; + std::atomic<uint64_t> total_requests_in_progress{0}; + std::atomic<uint64_t> total_error_requests{0}; + std::atomic<uint64_t> total_issued_requests{0}; + + // Protects backend_metrics. A mutex is necessary because the length of + // backend_metrics_ can be accessed by both the callback intercepting the + // call's recv_trailing_metadata and the load reporting thread. + Mutex backend_metrics_mu; + std::map<std::string, BackendMetric> backend_metrics + ABSL_GUARDED_BY(backend_metrics_mu); + }; + RefCountedPtr<XdsClient> xds_client_; const XdsBootstrap::XdsServer& lrs_server_; absl::string_view cluster_name_; absl::string_view eds_service_name_; RefCountedPtr<XdsLocalityName> name_; - - std::atomic<uint64_t> total_successful_requests_{0}; - std::atomic<uint64_t> total_requests_in_progress_{0}; - std::atomic<uint64_t> total_error_requests_{0}; - std::atomic<uint64_t> total_issued_requests_{0}; - - // Protects backend_metrics_. A mutex is necessary because the length of - // backend_metrics_ can be accessed by both the callback intercepting the - // call's recv_trailing_metadata (not from the control plane work serializer) - // and the load reporting thread (from the control plane work serializer). - Mutex backend_metrics_mu_; - std::map<std::string, BackendMetric> backend_metrics_ - ABSL_GUARDED_BY(backend_metrics_mu_); + PerCpu<Stats> stats_{PerCpuOptions().SetMaxShards(32).SetCpusPerShard(4)}; }; } // namespace grpc_core -#endif /* GRPC_CORE_EXT_XDS_XDS_CLIENT_STATS_H */ +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_CLIENT_STATS_H diff --git a/grpc/src/core/ext/xds/xds_cluster.cc b/grpc/src/core/ext/xds/xds_cluster.cc index 20b56317..ad3773e6 100644 --- a/grpc/src/core/ext/xds/xds_cluster.cc +++ b/grpc/src/core/ext/xds/xds_cluster.cc @@ -18,69 +18,112 @@ #include "src/core/ext/xds/xds_cluster.h" -#include "absl/container/inlined_vector.h" +#include <stddef.h> + +#include <utility> + +#include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" #include "absl/strings/str_join.h" +#include "absl/strings/strip.h" +#include "absl/types/variant.h" #include "envoy/config/cluster/v3/circuit_breaker.upb.h" #include "envoy/config/cluster/v3/cluster.upb.h" #include "envoy/config/cluster/v3/cluster.upbdefs.h" +#include "envoy/config/cluster/v3/outlier_detection.upb.h" #include "envoy/config/core/v3/address.upb.h" #include "envoy/config/core/v3/base.upb.h" #include "envoy/config/core/v3/config_source.upb.h" +#include "envoy/config/core/v3/health_check.upb.h" #include "envoy/config/endpoint/v3/endpoint.upb.h" #include "envoy/config/endpoint/v3/endpoint_components.upb.h" #include "envoy/extensions/clusters/aggregate/v3/cluster.upb.h" +#include "envoy/extensions/transport_sockets/tls/v3/tls.upb.h" #include "google/protobuf/any.upb.h" +#include "google/protobuf/duration.upb.h" #include "google/protobuf/wrappers.upb.h" +#include "upb/base/string_view.h" +#include "upb/text/encode.h" -#include <grpc/support/alloc.h> +#include <grpc/support/json.h> +#include <grpc/support/log.h> -#include "src/core/lib/gpr/env.h" +#include "src/core/ext/xds/upb_utils.h" +#include "src/core/ext/xds/xds_client.h" +#include "src/core/ext/xds/xds_common_types.h" +#include "src/core/ext/xds/xds_lb_policy_registry.h" +#include "src/core/ext/xds/xds_resource_type.h" +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/debug/trace.h" #include "src/core/lib/gpr/string.h" +#include "src/core/lib/gprpp/env.h" #include "src/core/lib/gprpp/host_port.h" +#include "src/core/lib/gprpp/match.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/gprpp/validation_errors.h" +#include "src/core/lib/json/json_writer.h" +#include "src/core/lib/load_balancing/lb_policy_registry.h" +#include "src/core/lib/matchers/matchers.h" namespace grpc_core { +// TODO(eostroukhov): Remove once this feature is no longer experimental. +bool XdsOverrideHostEnabled() { + auto value = GetEnv("GRPC_EXPERIMENTAL_XDS_ENABLE_OVERRIDE_HOST"); + if (!value.has_value()) return false; + bool parsed_value; + bool parse_succeeded = gpr_parse_bool_value(value->c_str(), &parsed_value); + return parse_succeeded && parsed_value; +} + // // XdsClusterResource // std::string XdsClusterResource::ToString() const { - absl::InlinedVector<std::string, 8> contents; - switch (cluster_type) { - case EDS: - contents.push_back("cluster_type=EDS"); - if (!eds_service_name.empty()) { - contents.push_back( - absl::StrFormat("eds_service_name=%s", eds_service_name)); - } - break; - case LOGICAL_DNS: - contents.push_back("cluster_type=LOGICAL_DNS"); - contents.push_back(absl::StrFormat("dns_hostname=%s", dns_hostname)); - break; - case AGGREGATE: - contents.push_back("cluster_type=AGGREGATE"); - contents.push_back( - absl::StrFormat("prioritized_cluster_names=[%s]", - absl::StrJoin(prioritized_cluster_names, ", "))); - } - if (!common_tls_context.Empty()) { - contents.push_back(absl::StrFormat("common_tls_context=%s", - common_tls_context.ToString())); - } + std::vector<std::string> contents; + Match( + type, + [&](const Eds& eds) { + contents.push_back("type=EDS"); + if (!eds.eds_service_name.empty()) { + contents.push_back( + absl::StrCat("eds_service_name=", eds.eds_service_name)); + } + }, + [&](const LogicalDns& logical_dns) { + contents.push_back("type=LOGICAL_DNS"); + contents.push_back(absl::StrCat("dns_hostname=", logical_dns.hostname)); + }, + [&](const Aggregate& aggregate) { + contents.push_back("type=AGGREGATE"); + contents.push_back(absl::StrCat( + "prioritized_cluster_names=[", + absl::StrJoin(aggregate.prioritized_cluster_names, ", "), "]")); + }); + contents.push_back(absl::StrCat("lb_policy_config=", + JsonDump(Json::FromArray(lb_policy_config)))); if (lrs_load_reporting_server.has_value()) { - contents.push_back(absl::StrFormat("lrs_load_reporting_server_name=%s", - lrs_load_reporting_server->server_uri)); + contents.push_back(absl::StrCat("lrs_load_reporting_server_name=", + lrs_load_reporting_server->server_uri())); } - contents.push_back(absl::StrCat("lb_policy=", lb_policy)); - if (lb_policy == "RING_HASH") { - contents.push_back(absl::StrCat("min_ring_size=", min_ring_size)); - contents.push_back(absl::StrCat("max_ring_size=", max_ring_size)); + if (!common_tls_context.Empty()) { + contents.push_back( + absl::StrCat("common_tls_context=", common_tls_context.ToString())); } contents.push_back( - absl::StrFormat("max_concurrent_requests=%d", max_concurrent_requests)); + absl::StrCat("max_concurrent_requests=", max_concurrent_requests)); + if (!override_host_statuses.empty()) { + std::vector<const char*> statuses; + statuses.reserve(override_host_statuses.size()); + for (const auto& status : override_host_statuses) { + statuses.push_back(status.ToString()); + } + contents.push_back(absl::StrCat("override_host_statuses={", + absl::StrJoin(statuses, ", "), "}")); + } return absl::StrCat("{", absl::StrJoin(contents, ", "), "}"); } @@ -90,287 +133,345 @@ std::string XdsClusterResource::ToString() const { namespace { -grpc_error_handle UpstreamTlsContextParse( - const XdsEncodingContext& context, +CommonTlsContext UpstreamTlsContextParse( + const XdsResourceType::DecodeContext& context, const envoy_config_core_v3_TransportSocket* transport_socket, - CommonTlsContext* common_tls_context) { - // Record Upstream tls context - absl::string_view name = UpbStringToAbsl( - envoy_config_core_v3_TransportSocket_name(transport_socket)); - if (name != "envoy.transport_sockets.tls") { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("Unrecognized transport socket: ", name)); - } - auto* typed_config = + ValidationErrors* errors) { + ValidationErrors::ScopedField field(errors, ".typed_config"); + const auto* typed_config = envoy_config_core_v3_TransportSocket_typed_config(transport_socket); - if (typed_config != nullptr) { - const upb_StringView encoded_upstream_tls_context = - google_protobuf_Any_value(typed_config); - auto* upstream_tls_context = - envoy_extensions_transport_sockets_tls_v3_UpstreamTlsContext_parse( - encoded_upstream_tls_context.data, - encoded_upstream_tls_context.size, context.arena); - if (upstream_tls_context == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Can't decode upstream tls context."); - } - auto* common_tls_context_proto = - envoy_extensions_transport_sockets_tls_v3_UpstreamTlsContext_common_tls_context( - upstream_tls_context); - if (common_tls_context_proto != nullptr) { - grpc_error_handle error = CommonTlsContext::Parse( - context, common_tls_context_proto, common_tls_context); - if (error != GRPC_ERROR_NONE) { - return grpc_error_add_child(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Error parsing UpstreamTlsContext"), - error); - } - } + auto extension = ExtractXdsExtension(context, typed_config, errors); + if (!extension.has_value()) return {}; + if (extension->type != + "envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext") { + ValidationErrors::ScopedField field(errors, ".type_url"); + errors->AddError("unsupported transport socket type"); + return {}; + } + absl::string_view* serialized_upstream_tls_context = + absl::get_if<absl::string_view>(&extension->value); + if (serialized_upstream_tls_context == nullptr) { + errors->AddError("can't decode UpstreamTlsContext"); + return {}; } - if (common_tls_context->certificate_validation_context + const auto* upstream_tls_context = + envoy_extensions_transport_sockets_tls_v3_UpstreamTlsContext_parse( + serialized_upstream_tls_context->data(), + serialized_upstream_tls_context->size(), context.arena); + if (upstream_tls_context == nullptr) { + errors->AddError("can't decode UpstreamTlsContext"); + return {}; + } + ValidationErrors::ScopedField field3(errors, ".common_tls_context"); + const auto* common_tls_context_proto = + envoy_extensions_transport_sockets_tls_v3_UpstreamTlsContext_common_tls_context( + upstream_tls_context); + CommonTlsContext common_tls_context; + if (common_tls_context_proto != nullptr) { + common_tls_context = + CommonTlsContext::Parse(context, common_tls_context_proto, errors); + } + if (common_tls_context.certificate_validation_context .ca_certificate_provider_instance.instance_name.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "UpstreamTlsContext: TLS configuration provided but no " - "ca_certificate_provider_instance found."); + errors->AddError("no CA certificate provider instance configured"); } - return GRPC_ERROR_NONE; + return common_tls_context; } -grpc_error_handle CdsLogicalDnsParse( - const envoy_config_cluster_v3_Cluster* cluster, - XdsClusterResource* cds_update) { +XdsClusterResource::Eds EdsConfigParse( + const envoy_config_cluster_v3_Cluster* cluster, ValidationErrors* errors) { + XdsClusterResource::Eds eds; + ValidationErrors::ScopedField field(errors, ".eds_cluster_config"); + const envoy_config_cluster_v3_Cluster_EdsClusterConfig* eds_cluster_config = + envoy_config_cluster_v3_Cluster_eds_cluster_config(cluster); + if (eds_cluster_config == nullptr) { + errors->AddError("field not present"); + } else { + ValidationErrors::ScopedField field(errors, ".eds_config"); + const envoy_config_core_v3_ConfigSource* eds_config = + envoy_config_cluster_v3_Cluster_EdsClusterConfig_eds_config( + eds_cluster_config); + if (eds_config == nullptr) { + errors->AddError("field not present"); + } else { + if (!envoy_config_core_v3_ConfigSource_has_ads(eds_config) && + !envoy_config_core_v3_ConfigSource_has_self(eds_config)) { + errors->AddError("ConfigSource is not ads or self"); + } + // Record EDS service_name (if any). + upb_StringView service_name = + envoy_config_cluster_v3_Cluster_EdsClusterConfig_service_name( + eds_cluster_config); + if (service_name.size != 0) { + eds.eds_service_name = UpbStringToStdString(service_name); + } + } + } + return eds; +} + +XdsClusterResource::LogicalDns LogicalDnsParse( + const envoy_config_cluster_v3_Cluster* cluster, ValidationErrors* errors) { + XdsClusterResource::LogicalDns logical_dns; + ValidationErrors::ScopedField field(errors, ".load_assignment"); const auto* load_assignment = envoy_config_cluster_v3_Cluster_load_assignment(cluster); if (load_assignment == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "load_assignment not present for LOGICAL_DNS cluster"); + errors->AddError("field not present for LOGICAL_DNS cluster"); + return logical_dns; } + ValidationErrors::ScopedField field2(errors, ".endpoints"); size_t num_localities; const auto* const* localities = envoy_config_endpoint_v3_ClusterLoadAssignment_endpoints(load_assignment, &num_localities); if (num_localities != 1) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("load_assignment for LOGICAL_DNS cluster must have " - "exactly one locality, found ", - num_localities)); + errors->AddError(absl::StrCat( + "must contain exactly one locality for LOGICAL_DNS cluster, found ", + num_localities)); + return logical_dns; } + ValidationErrors::ScopedField field3(errors, "[0].lb_endpoints"); size_t num_endpoints; const auto* const* endpoints = envoy_config_endpoint_v3_LocalityLbEndpoints_lb_endpoints(localities[0], &num_endpoints); if (num_endpoints != 1) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("locality for LOGICAL_DNS cluster must have " - "exactly one endpoint, found ", - num_endpoints)); + errors->AddError(absl::StrCat( + "must contain exactly one endpoint for LOGICAL_DNS cluster, found ", + num_endpoints)); + return logical_dns; } + ValidationErrors::ScopedField field4(errors, "[0].endpoint"); const auto* endpoint = envoy_config_endpoint_v3_LbEndpoint_endpoint(endpoints[0]); if (endpoint == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "LbEndpoint endpoint field not set"); + errors->AddError("field not present"); + return logical_dns; } + ValidationErrors::ScopedField field5(errors, ".address"); const auto* address = envoy_config_endpoint_v3_Endpoint_address(endpoint); if (address == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Endpoint address field not set"); + errors->AddError("field not present"); + return logical_dns; } + ValidationErrors::ScopedField field6(errors, ".socket_address"); const auto* socket_address = envoy_config_core_v3_Address_socket_address(address); if (socket_address == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Address socket_address field not set"); + errors->AddError("field not present"); + return logical_dns; } if (envoy_config_core_v3_SocketAddress_resolver_name(socket_address).size != 0) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + ValidationErrors::ScopedField field(errors, ".resolver_name"); + errors->AddError( "LOGICAL_DNS clusters must NOT have a custom resolver name set"); } absl::string_view address_str = UpbStringToAbsl( envoy_config_core_v3_SocketAddress_address(socket_address)); if (address_str.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "SocketAddress address field not set"); + ValidationErrors::ScopedField field(errors, ".address"); + errors->AddError("field not present"); } if (!envoy_config_core_v3_SocketAddress_has_port_value(socket_address)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "SocketAddress port_value field not set"); + ValidationErrors::ScopedField field(errors, ".port_value"); + errors->AddError("field not present"); } - cds_update->dns_hostname = JoinHostPort( + logical_dns.hostname = JoinHostPort( address_str, envoy_config_core_v3_SocketAddress_port_value(socket_address)); - return GRPC_ERROR_NONE; + return logical_dns; } -// TODO(donnadionne): Check to see if cluster types aggregate_cluster and -// logical_dns are enabled, this will be -// removed once the cluster types are fully integration-tested and enabled by -// default. -bool XdsAggregateAndLogicalDnsClusterEnabled() { - char* value = gpr_getenv( - "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER"); - bool parsed_value; - bool parse_succeeded = gpr_parse_bool_value(value, &parsed_value); - gpr_free(value); - return parse_succeeded && parsed_value; +XdsClusterResource::Aggregate AggregateClusterParse( + const XdsResourceType::DecodeContext& context, + absl::string_view serialized_config, ValidationErrors* errors) { + XdsClusterResource::Aggregate aggregate; + const auto* aggregate_cluster_config = + envoy_extensions_clusters_aggregate_v3_ClusterConfig_parse( + serialized_config.data(), serialized_config.size(), context.arena); + if (aggregate_cluster_config == nullptr) { + errors->AddError("can't parse aggregate cluster config"); + return aggregate; + } + size_t size; + const upb_StringView* clusters = + envoy_extensions_clusters_aggregate_v3_ClusterConfig_clusters( + aggregate_cluster_config, &size); + if (size == 0) { + ValidationErrors::ScopedField field(errors, ".clusters"); + errors->AddError("must be non-empty"); + } + for (size_t i = 0; i < size; ++i) { + aggregate.prioritized_cluster_names.emplace_back( + UpbStringToStdString(clusters[i])); + } + return aggregate; } -grpc_error_handle CdsResourceParse( - const XdsEncodingContext& context, - const envoy_config_cluster_v3_Cluster* cluster, bool /*is_v2*/, - XdsClusterResource* cds_update) { - std::vector<grpc_error_handle> errors; - // Check the cluster_discovery_type. - if (!envoy_config_cluster_v3_Cluster_has_type(cluster) && - !envoy_config_cluster_v3_Cluster_has_cluster_type(cluster)) { - errors.push_back( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("DiscoveryType not found.")); - } else if (envoy_config_cluster_v3_Cluster_type(cluster) == - envoy_config_cluster_v3_Cluster_EDS) { - cds_update->cluster_type = XdsClusterResource::ClusterType::EDS; - // Check the EDS config source. - const envoy_config_cluster_v3_Cluster_EdsClusterConfig* eds_cluster_config = - envoy_config_cluster_v3_Cluster_eds_cluster_config(cluster); - const envoy_config_core_v3_ConfigSource* eds_config = - envoy_config_cluster_v3_Cluster_EdsClusterConfig_eds_config( - eds_cluster_config); - if (!envoy_config_core_v3_ConfigSource_has_ads(eds_config) && - !envoy_config_core_v3_ConfigSource_has_self(eds_config)) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "EDS ConfigSource is not ADS or SELF.")); - } - // Record EDS service_name (if any). - upb_StringView service_name = - envoy_config_cluster_v3_Cluster_EdsClusterConfig_service_name( - eds_cluster_config); - if (service_name.size != 0) { - cds_update->eds_service_name = UpbStringToStdString(service_name); - } - } else if (!XdsAggregateAndLogicalDnsClusterEnabled()) { - errors.push_back( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("DiscoveryType is not valid.")); - } else if (envoy_config_cluster_v3_Cluster_type(cluster) == - envoy_config_cluster_v3_Cluster_LOGICAL_DNS) { - cds_update->cluster_type = XdsClusterResource::ClusterType::LOGICAL_DNS; - grpc_error_handle error = CdsLogicalDnsParse(cluster, cds_update); - if (error != GRPC_ERROR_NONE) errors.push_back(error); - } else { - if (!envoy_config_cluster_v3_Cluster_has_cluster_type(cluster)) { - errors.push_back( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("DiscoveryType is not valid.")); - } else { - const envoy_config_cluster_v3_Cluster_CustomClusterType* - custom_cluster_type = - envoy_config_cluster_v3_Cluster_cluster_type(cluster); - upb_StringView type_name = - envoy_config_cluster_v3_Cluster_CustomClusterType_name( - custom_cluster_type); - if (UpbStringToAbsl(type_name) != "envoy.clusters.aggregate") { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "DiscoveryType is not valid.")); - } else { - cds_update->cluster_type = XdsClusterResource::ClusterType::AGGREGATE; - // Retrieve aggregate clusters. - const google_protobuf_Any* typed_config = - envoy_config_cluster_v3_Cluster_CustomClusterType_typed_config( - custom_cluster_type); - const upb_StringView aggregate_cluster_config_upb_stringview = - google_protobuf_Any_value(typed_config); - const envoy_extensions_clusters_aggregate_v3_ClusterConfig* - aggregate_cluster_config = - envoy_extensions_clusters_aggregate_v3_ClusterConfig_parse( - aggregate_cluster_config_upb_stringview.data, - aggregate_cluster_config_upb_stringview.size, - context.arena); - if (aggregate_cluster_config == nullptr) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Can't parse aggregate cluster.")); - } else { - size_t size; - const upb_StringView* clusters = - envoy_extensions_clusters_aggregate_v3_ClusterConfig_clusters( - aggregate_cluster_config, &size); - for (size_t i = 0; i < size; ++i) { - const upb_StringView cluster = clusters[i]; - cds_update->prioritized_cluster_names.emplace_back( - UpbStringToStdString(cluster)); - } - } - } +void ParseLbPolicyConfig(const XdsResourceType::DecodeContext& context, + const envoy_config_cluster_v3_Cluster* cluster, + XdsClusterResource* cds_update, + ValidationErrors* errors) { + // First, check the new load_balancing_policy field. + const auto* load_balancing_policy = + envoy_config_cluster_v3_Cluster_load_balancing_policy(cluster); + if (load_balancing_policy != nullptr) { + const auto& registry = + static_cast<const GrpcXdsBootstrap&>(context.client->bootstrap()) + .lb_policy_registry(); + ValidationErrors::ScopedField field(errors, ".load_balancing_policy"); + const size_t original_error_count = errors->size(); + cds_update->lb_policy_config = registry.ConvertXdsLbPolicyConfig( + context, load_balancing_policy, errors); + // If there were no conversion errors, validate that the converted config + // parses with the gRPC LB policy registry. + if (original_error_count == errors->size()) { + auto config = CoreConfiguration::Get() + .lb_policy_registry() + .ParseLoadBalancingConfig( + Json::FromArray(cds_update->lb_policy_config)); + if (!config.ok()) errors->AddError(config.status().message()); } + return; } - // Check the LB policy. + // Didn't find load_balancing_policy field, so fall back to the old + // lb_policy enum field. if (envoy_config_cluster_v3_Cluster_lb_policy(cluster) == envoy_config_cluster_v3_Cluster_ROUND_ROBIN) { - cds_update->lb_policy = "ROUND_ROBIN"; + cds_update->lb_policy_config = { + Json::FromObject({ + {"xds_wrr_locality_experimental", + Json::FromObject({ + {"childPolicy", Json::FromArray({ + Json::FromObject({ + {"round_robin", Json::FromObject({})}, + }), + })}, + })}, + }), + }; } else if (envoy_config_cluster_v3_Cluster_lb_policy(cluster) == envoy_config_cluster_v3_Cluster_RING_HASH) { - cds_update->lb_policy = "RING_HASH"; // Record ring hash lb config auto* ring_hash_config = envoy_config_cluster_v3_Cluster_ring_hash_lb_config(cluster); + uint64_t min_ring_size = 1024; + uint64_t max_ring_size = 8388608; if (ring_hash_config != nullptr) { - const google_protobuf_UInt64Value* max_ring_size = + ValidationErrors::ScopedField field(errors, ".ring_hash_lb_config"); + const google_protobuf_UInt64Value* uint64_value = envoy_config_cluster_v3_Cluster_RingHashLbConfig_maximum_ring_size( ring_hash_config); - if (max_ring_size != nullptr) { - cds_update->max_ring_size = - google_protobuf_UInt64Value_value(max_ring_size); - if (cds_update->max_ring_size > 8388608 || - cds_update->max_ring_size == 0) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "max_ring_size is not in the range of 1 to 8388608.")); + if (uint64_value != nullptr) { + ValidationErrors::ScopedField field(errors, ".maximum_ring_size"); + max_ring_size = google_protobuf_UInt64Value_value(uint64_value); + if (max_ring_size > 8388608 || max_ring_size == 0) { + errors->AddError("must be in the range of 1 to 8388608"); } } - const google_protobuf_UInt64Value* min_ring_size = + uint64_value = envoy_config_cluster_v3_Cluster_RingHashLbConfig_minimum_ring_size( ring_hash_config); - if (min_ring_size != nullptr) { - cds_update->min_ring_size = - google_protobuf_UInt64Value_value(min_ring_size); - if (cds_update->min_ring_size > 8388608 || - cds_update->min_ring_size == 0) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "min_ring_size is not in the range of 1 to 8388608.")); + if (uint64_value != nullptr) { + ValidationErrors::ScopedField field(errors, ".minimum_ring_size"); + min_ring_size = google_protobuf_UInt64Value_value(uint64_value); + if (min_ring_size > 8388608 || min_ring_size == 0) { + errors->AddError("must be in the range of 1 to 8388608"); } - if (cds_update->min_ring_size > cds_update->max_ring_size) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "min_ring_size cannot be greater than max_ring_size.")); + if (min_ring_size > max_ring_size) { + errors->AddError("cannot be greater than maximum_ring_size"); } } if (envoy_config_cluster_v3_Cluster_RingHashLbConfig_hash_function( ring_hash_config) != envoy_config_cluster_v3_Cluster_RingHashLbConfig_XX_HASH) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "ring hash lb config has invalid hash function.")); + ValidationErrors::ScopedField field(errors, ".hash_function"); + errors->AddError("invalid hash function"); } } + cds_update->lb_policy_config = { + Json::FromObject({ + {"ring_hash_experimental", + Json::FromObject({ + {"minRingSize", Json::FromNumber(min_ring_size)}, + {"maxRingSize", Json::FromNumber(max_ring_size)}, + })}, + }), + }; } else { - errors.push_back( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("LB policy is not supported.")); + ValidationErrors::ScopedField field(errors, ".lb_policy"); + errors->AddError("LB policy is not supported"); } +} + +absl::StatusOr<XdsClusterResource> CdsResourceParse( + const XdsResourceType::DecodeContext& context, + const envoy_config_cluster_v3_Cluster* cluster) { + XdsClusterResource cds_update; + ValidationErrors errors; + // Check the cluster discovery type. + if (envoy_config_cluster_v3_Cluster_type(cluster) == + envoy_config_cluster_v3_Cluster_EDS) { + cds_update.type = EdsConfigParse(cluster, &errors); + } else if (envoy_config_cluster_v3_Cluster_type(cluster) == + envoy_config_cluster_v3_Cluster_LOGICAL_DNS) { + cds_update.type = LogicalDnsParse(cluster, &errors); + } else if (envoy_config_cluster_v3_Cluster_has_cluster_type(cluster)) { + ValidationErrors::ScopedField field(&errors, ".cluster_type"); + const auto* custom_cluster_type = + envoy_config_cluster_v3_Cluster_cluster_type(cluster); + GPR_ASSERT(custom_cluster_type != nullptr); + ValidationErrors::ScopedField field2(&errors, ".typed_config"); + const auto* typed_config = + envoy_config_cluster_v3_Cluster_CustomClusterType_typed_config( + custom_cluster_type); + if (typed_config == nullptr) { + errors.AddError("field not present"); + } else { + absl::string_view type_url = absl::StripPrefix( + UpbStringToAbsl(google_protobuf_Any_type_url(typed_config)), + "type.googleapis.com/"); + if (type_url != "envoy.extensions.clusters.aggregate.v3.ClusterConfig") { + ValidationErrors::ScopedField field(&errors, ".type_url"); + errors.AddError( + absl::StrCat("unknown cluster_type extension: ", type_url)); + } else { + // Retrieve aggregate clusters. + ValidationErrors::ScopedField field( + &errors, + ".value[envoy.extensions.clusters.aggregate.v3.ClusterConfig]"); + absl::string_view serialized_config = + UpbStringToAbsl(google_protobuf_Any_value(typed_config)); + cds_update.type = + AggregateClusterParse(context, serialized_config, &errors); + } + } + } else { + ValidationErrors::ScopedField field(&errors, ".type"); + errors.AddError("unknown discovery type"); + } + // Check the LB policy. + ParseLbPolicyConfig(context, cluster, &cds_update, &errors); + // transport_socket auto* transport_socket = envoy_config_cluster_v3_Cluster_transport_socket(cluster); if (transport_socket != nullptr) { - grpc_error_handle error = UpstreamTlsContextParse( - context, transport_socket, &cds_update->common_tls_context); - if (error != GRPC_ERROR_NONE) { - errors.push_back( - grpc_error_add_child(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Error parsing security configuration"), - error)); - } + ValidationErrors::ScopedField field(&errors, ".transport_socket"); + cds_update.common_tls_context = + UpstreamTlsContextParse(context, transport_socket, &errors); } // Record LRS server name (if any). const envoy_config_core_v3_ConfigSource* lrs_server = envoy_config_cluster_v3_Cluster_lrs_server(cluster); if (lrs_server != nullptr) { if (!envoy_config_core_v3_ConfigSource_has_self(lrs_server)) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - ": LRS ConfigSource is not self.")); + ValidationErrors::ScopedField field(&errors, ".lrs_server"); + errors.AddError("ConfigSource is not self"); } - cds_update->lrs_load_reporting_server.emplace(context.server); + cds_update.lrs_load_reporting_server.emplace( + static_cast<const GrpcXdsBootstrap::GrpcXdsServer&>(context.server)); } // The Cluster resource encodes the circuit breaking parameters in a list of // Thresholds messages, where each message specifies the parameters for a @@ -391,17 +492,167 @@ grpc_error_handle CdsResourceParse( envoy_config_cluster_v3_CircuitBreakers_Thresholds_max_requests( threshold); if (max_requests != nullptr) { - cds_update->max_concurrent_requests = + cds_update.max_concurrent_requests = google_protobuf_UInt32Value_value(max_requests); } break; } } } - return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing CDS resource", &errors); + // Outlier detection config. + if (envoy_config_cluster_v3_Cluster_has_outlier_detection(cluster)) { + ValidationErrors::ScopedField field(&errors, ".outlier_detection"); + OutlierDetectionConfig outlier_detection_update; + const envoy_config_cluster_v3_OutlierDetection* outlier_detection = + envoy_config_cluster_v3_Cluster_outlier_detection(cluster); + const google_protobuf_Duration* duration = + envoy_config_cluster_v3_OutlierDetection_interval(outlier_detection); + if (duration != nullptr) { + ValidationErrors::ScopedField field(&errors, ".interval"); + outlier_detection_update.interval = ParseDuration(duration, &errors); + } + duration = envoy_config_cluster_v3_OutlierDetection_base_ejection_time( + outlier_detection); + if (duration != nullptr) { + ValidationErrors::ScopedField field(&errors, ".base_ejection_time"); + outlier_detection_update.base_ejection_time = + ParseDuration(duration, &errors); + } + duration = envoy_config_cluster_v3_OutlierDetection_max_ejection_time( + outlier_detection); + if (duration != nullptr) { + ValidationErrors::ScopedField field(&errors, ".max_ejection_time"); + outlier_detection_update.max_ejection_time = + ParseDuration(duration, &errors); + } + const google_protobuf_UInt32Value* max_ejection_percent = + envoy_config_cluster_v3_OutlierDetection_max_ejection_percent( + outlier_detection); + if (max_ejection_percent != nullptr) { + outlier_detection_update.max_ejection_percent = + google_protobuf_UInt32Value_value(max_ejection_percent); + if (outlier_detection_update.max_ejection_percent > 100) { + ValidationErrors::ScopedField field(&errors, ".max_ejection_percent"); + errors.AddError("value must be <= 100"); + } + } + const google_protobuf_UInt32Value* enforcing_success_rate = + envoy_config_cluster_v3_OutlierDetection_enforcing_success_rate( + outlier_detection); + if (enforcing_success_rate != nullptr) { + uint32_t enforcement_percentage = + google_protobuf_UInt32Value_value(enforcing_success_rate); + if (enforcement_percentage > 100) { + ValidationErrors::ScopedField field(&errors, ".enforcing_success_rate"); + errors.AddError("value must be <= 100"); + } + if (enforcement_percentage != 0) { + OutlierDetectionConfig::SuccessRateEjection success_rate_ejection; + success_rate_ejection.enforcement_percentage = enforcement_percentage; + const google_protobuf_UInt32Value* minimum_hosts = + envoy_config_cluster_v3_OutlierDetection_success_rate_minimum_hosts( + outlier_detection); + if (minimum_hosts != nullptr) { + success_rate_ejection.minimum_hosts = + google_protobuf_UInt32Value_value(minimum_hosts); + } + const google_protobuf_UInt32Value* request_volume = + envoy_config_cluster_v3_OutlierDetection_success_rate_request_volume( + outlier_detection); + if (request_volume != nullptr) { + success_rate_ejection.request_volume = + google_protobuf_UInt32Value_value(request_volume); + } + const google_protobuf_UInt32Value* stdev_factor = + envoy_config_cluster_v3_OutlierDetection_success_rate_stdev_factor( + outlier_detection); + if (stdev_factor != nullptr) { + success_rate_ejection.stdev_factor = + google_protobuf_UInt32Value_value(stdev_factor); + } + outlier_detection_update.success_rate_ejection = success_rate_ejection; + } + } + const google_protobuf_UInt32Value* enforcing_failure_percentage = + envoy_config_cluster_v3_OutlierDetection_enforcing_failure_percentage( + outlier_detection); + if (enforcing_failure_percentage != nullptr) { + uint32_t enforcement_percentage = + google_protobuf_UInt32Value_value(enforcing_failure_percentage); + if (enforcement_percentage > 100) { + ValidationErrors::ScopedField field(&errors, + ".enforcing_failure_percentage"); + errors.AddError("value must be <= 100"); + } + if (enforcement_percentage != 0) { + OutlierDetectionConfig::FailurePercentageEjection + failure_percentage_ejection; + failure_percentage_ejection.enforcement_percentage = + enforcement_percentage; + const google_protobuf_UInt32Value* minimum_hosts = + envoy_config_cluster_v3_OutlierDetection_failure_percentage_minimum_hosts( + outlier_detection); + if (minimum_hosts != nullptr) { + failure_percentage_ejection.minimum_hosts = + google_protobuf_UInt32Value_value(minimum_hosts); + } + const google_protobuf_UInt32Value* request_volume = + envoy_config_cluster_v3_OutlierDetection_failure_percentage_request_volume( + outlier_detection); + if (request_volume != nullptr) { + failure_percentage_ejection.request_volume = + google_protobuf_UInt32Value_value(request_volume); + } + const google_protobuf_UInt32Value* threshold = + envoy_config_cluster_v3_OutlierDetection_failure_percentage_threshold( + outlier_detection); + if (threshold != nullptr) { + failure_percentage_ejection.threshold = + google_protobuf_UInt32Value_value(threshold); + if (enforcement_percentage > 100) { + ValidationErrors::ScopedField field( + &errors, ".failure_percentage_threshold"); + errors.AddError("value must be <= 100"); + } + } + outlier_detection_update.failure_percentage_ejection = + failure_percentage_ejection; + } + } + cds_update.outlier_detection = outlier_detection_update; + } + // Validate override host status. + if (XdsOverrideHostEnabled()) { + const auto* common_lb_config = + envoy_config_cluster_v3_Cluster_common_lb_config(cluster); + if (common_lb_config != nullptr) { + ValidationErrors::ScopedField field(&errors, ".common_lb_config"); + const auto* override_host_status = + envoy_config_cluster_v3_Cluster_CommonLbConfig_override_host_status( + common_lb_config); + if (override_host_status != nullptr) { + ValidationErrors::ScopedField field(&errors, ".override_host_status"); + size_t size; + const int32_t* statuses = envoy_config_core_v3_HealthStatusSet_statuses( + override_host_status, &size); + for (size_t i = 0; i < size; ++i) { + auto status = XdsHealthStatus::FromUpb(statuses[i]); + if (status.has_value()) { + cds_update.override_host_statuses.insert(*status); + } + } + } + } + } + // Return result. + if (!errors.ok()) { + return errors.status(absl::StatusCode::kInvalidArgument, + "errors validating Cluster resource"); + } + return cds_update; } -void MaybeLogCluster(const XdsEncodingContext& context, +void MaybeLogCluster(const XdsResourceType::DecodeContext& context, const envoy_config_cluster_v3_Cluster* cluster) { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { @@ -415,39 +666,39 @@ void MaybeLogCluster(const XdsEncodingContext& context, } // namespace -absl::StatusOr<XdsResourceType::DecodeResult> XdsClusterResourceType::Decode( - const XdsEncodingContext& context, absl::string_view serialized_resource, - bool is_v2) const { +XdsResourceType::DecodeResult XdsClusterResourceType::Decode( + const XdsResourceType::DecodeContext& context, + absl::string_view serialized_resource) const { + DecodeResult result; // Parse serialized proto. auto* resource = envoy_config_cluster_v3_Cluster_parse( serialized_resource.data(), serialized_resource.size(), context.arena); if (resource == nullptr) { - return absl::InvalidArgumentError("Can't parse Cluster resource."); + result.resource = + absl::InvalidArgumentError("Can't parse Cluster resource."); + return result; } MaybeLogCluster(context, resource); // Validate resource. - DecodeResult result; result.name = UpbStringToStdString(envoy_config_cluster_v3_Cluster_name(resource)); - auto cluster_data = absl::make_unique<ResourceDataSubclass>(); - grpc_error_handle error = - CdsResourceParse(context, resource, is_v2, &cluster_data->resource); - if (error != GRPC_ERROR_NONE) { - std::string error_str = grpc_error_std_string(error); - GRPC_ERROR_UNREF(error); + auto cds_resource = CdsResourceParse(context, resource); + if (!cds_resource.ok()) { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer)) { gpr_log(GPR_ERROR, "[xds_client %p] invalid Cluster %s: %s", - context.client, result.name.c_str(), error_str.c_str()); + context.client, result.name->c_str(), + cds_resource.status().ToString().c_str()); } - result.resource = absl::InvalidArgumentError(error_str); + result.resource = cds_resource.status(); } else { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer)) { gpr_log(GPR_INFO, "[xds_client %p] parsed Cluster %s: %s", context.client, - result.name.c_str(), cluster_data->resource.ToString().c_str()); + result.name->c_str(), cds_resource->ToString().c_str()); } - result.resource = std::move(cluster_data); + result.resource = + std::make_unique<XdsClusterResource>(std::move(*cds_resource)); } - return std::move(result); + return result; } } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_cluster.h b/grpc/src/core/ext/xds/xds_cluster.h index 5ac020fe..878f7954 100644 --- a/grpc/src/core/ext/xds/xds_cluster.h +++ b/grpc/src/core/ext/xds/xds_cluster.h @@ -14,66 +14,98 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_CLUSTER_H -#define GRPC_CORE_EXT_XDS_XDS_CLUSTER_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_CLUSTER_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_CLUSTER_H #include <grpc/support/port_platform.h> +#include <stdint.h> + +#include <algorithm> +#include <memory> +#include <set> #include <string> #include <vector> +#include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "absl/types/variant.h" #include "envoy/config/cluster/v3/cluster.upbdefs.h" #include "envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.h" #include "envoy/extensions/transport_sockets/tls/v3/tls.upbdefs.h" +#include "upb/reflection/def.h" +#include "src/core/ext/filters/client_channel/lb_policy/outlier_detection/outlier_detection.h" +#include "src/core/ext/xds/xds_bootstrap.h" +#include "src/core/ext/xds/xds_bootstrap_grpc.h" #include "src/core/ext/xds/xds_client.h" #include "src/core/ext/xds/xds_common_types.h" +#include "src/core/ext/xds/xds_health_status.h" +#include "src/core/ext/xds/xds_resource_type.h" #include "src/core/ext/xds/xds_resource_type_impl.h" +#include "src/core/lib/json/json.h" namespace grpc_core { -struct XdsClusterResource { - enum ClusterType { EDS, LOGICAL_DNS, AGGREGATE }; - ClusterType cluster_type; - // For cluster type EDS. - // The name to use in the EDS request. - // If empty, the cluster name will be used. - std::string eds_service_name; - // For cluster type LOGICAL_DNS. - // The hostname to lookup in DNS. - std::string dns_hostname; - // For cluster type AGGREGATE. - // The prioritized list of cluster names. - std::vector<std::string> prioritized_cluster_names; +bool XdsOverrideHostEnabled(); - // Tls Context used by clients - CommonTlsContext common_tls_context; +struct XdsClusterResource : public XdsResourceType::ResourceData { + struct Eds { + // If empty, defaults to the cluster name. + std::string eds_service_name; + + bool operator==(const Eds& other) const { + return eds_service_name == other.eds_service_name; + } + }; + + struct LogicalDns { + // The hostname to lookup in DNS. + std::string hostname; + + bool operator==(const LogicalDns& other) const { + return hostname == other.hostname; + } + }; + + struct Aggregate { + // Prioritized list of cluster names. + std::vector<std::string> prioritized_cluster_names; + + bool operator==(const Aggregate& other) const { + return prioritized_cluster_names == other.prioritized_cluster_names; + } + }; + + absl::variant<Eds, LogicalDns, Aggregate> type; + + // The LB policy to use for locality and endpoint picking. + Json::Array lb_policy_config; + + // Note: Remaining fields are not used for aggregate clusters. // The LRS server to use for load reporting. // If not set, load reporting will be disabled. - absl::optional<XdsBootstrap::XdsServer> lrs_load_reporting_server; + absl::optional<GrpcXdsBootstrap::GrpcXdsServer> lrs_load_reporting_server; + + // Tls Context used by clients + CommonTlsContext common_tls_context; - // The LB policy to use (e.g., "ROUND_ROBIN" or "RING_HASH"). - std::string lb_policy; - // Used for RING_HASH LB policy only. - uint64_t min_ring_size = 1024; - uint64_t max_ring_size = 8388608; // Maximum number of outstanding requests can be made to the upstream // cluster. uint32_t max_concurrent_requests = 1024; + absl::optional<OutlierDetectionConfig> outlier_detection; + + std::set<XdsHealthStatus> override_host_statuses; + bool operator==(const XdsClusterResource& other) const { - return cluster_type == other.cluster_type && - eds_service_name == other.eds_service_name && - dns_hostname == other.dns_hostname && - prioritized_cluster_names == other.prioritized_cluster_names && - common_tls_context == other.common_tls_context && + return type == other.type && lb_policy_config == other.lb_policy_config && lrs_load_reporting_server == other.lrs_load_reporting_server && - lb_policy == other.lb_policy && - min_ring_size == other.min_ring_size && - max_ring_size == other.max_ring_size && - max_concurrent_requests == other.max_concurrent_requests; + common_tls_context == other.common_tls_context && + max_concurrent_requests == other.max_concurrent_requests && + outlier_detection == other.outlier_detection && + override_host_statuses == other.override_host_statuses; } std::string ToString() const; @@ -85,17 +117,13 @@ class XdsClusterResourceType absl::string_view type_url() const override { return "envoy.config.cluster.v3.Cluster"; } - absl::string_view v2_type_url() const override { - return "envoy.api.v2.Cluster"; - } - absl::StatusOr<DecodeResult> Decode(const XdsEncodingContext& context, - absl::string_view serialized_resource, - bool is_v2) const override; + DecodeResult Decode(const XdsResourceType::DecodeContext& context, + absl::string_view serialized_resource) const override; bool AllResourcesRequiredInSotW() const override { return true; } - void InitUpbSymtab(upb_DefPool* symtab) const override { + void InitUpbSymtab(XdsClient*, upb_DefPool* symtab) const override { envoy_config_cluster_v3_Cluster_getmsgdef(symtab); envoy_extensions_clusters_aggregate_v3_ClusterConfig_getmsgdef(symtab); envoy_extensions_transport_sockets_tls_v3_UpstreamTlsContext_getmsgdef( @@ -105,4 +133,4 @@ class XdsClusterResourceType } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_CLUSTER_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_CLUSTER_H diff --git a/grpc/src/core/ext/xds/xds_cluster_specifier_plugin.cc b/grpc/src/core/ext/xds/xds_cluster_specifier_plugin.cc index e13f2283..97b6229d 100644 --- a/grpc/src/core/ext/xds/xds_cluster_specifier_plugin.cc +++ b/grpc/src/core/ext/xds/xds_cluster_specifier_plugin.cc @@ -18,42 +18,64 @@ #include "src/core/ext/xds/xds_cluster_specifier_plugin.h" -#include "absl/strings/str_format.h" -#include "envoy/extensions/filters/http/router/v3/router.upb.h" -#include "envoy/extensions/filters/http/router/v3/router.upbdefs.h" -#include "google/protobuf/duration.upb.h" -#include "upb/json_encode.h" - -#include "src/core/ext/filters/client_channel/lb_policy_registry.h" -#include "src/core/ext/xds/upb_utils.h" +#include <stddef.h> + +#include <map> +#include <utility> + +#include "absl/status/statusor.h" +#include "absl/strings/str_cat.h" +#include "absl/types/variant.h" +#include "upb/base/status.h" +#include "upb/json/encode.h" +#include "upb/upb.hpp" + +#include <grpc/support/json.h> +#include <grpc/support/log.h> + +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_reader.h" #include "src/proto/grpc/lookup/v1/rls_config.upb.h" #include "src/proto/grpc/lookup/v1/rls_config.upbdefs.h" namespace grpc_core { -const char* kXdsRouteLookupClusterSpecifierPluginConfigName = - "grpc.lookup.v1.RouteLookupClusterSpecifier"; +// +// XdsRouteLookupClusterSpecifierPlugin +// + +absl::string_view XdsRouteLookupClusterSpecifierPlugin::ConfigProtoName() + const { + return "grpc.lookup.v1.RouteLookupClusterSpecifier"; +} void XdsRouteLookupClusterSpecifierPlugin::PopulateSymtab( upb_DefPool* symtab) const { grpc_lookup_v1_RouteLookupConfig_getmsgdef(symtab); } -absl::StatusOr<std::string> -XdsRouteLookupClusterSpecifierPlugin::GenerateLoadBalancingPolicyConfig( - upb_StringView serialized_plugin_config, upb_Arena* arena, - upb_DefPool* symtab) const { +Json XdsRouteLookupClusterSpecifierPlugin::GenerateLoadBalancingPolicyConfig( + XdsExtension extension, upb_Arena* arena, upb_DefPool* symtab, + ValidationErrors* errors) const { + absl::string_view* serialized_plugin_config = + absl::get_if<absl::string_view>(&extension.value); + if (serialized_plugin_config == nullptr) { + errors->AddError("could not parse plugin config"); + return {}; + } const auto* specifier = grpc_lookup_v1_RouteLookupClusterSpecifier_parse( - serialized_plugin_config.data, serialized_plugin_config.size, arena); + serialized_plugin_config->data(), serialized_plugin_config->size(), + arena); if (specifier == nullptr) { - return absl::InvalidArgumentError("Could not parse plugin config"); + errors->AddError("could not parse plugin config"); + return {}; } const auto* plugin_config = grpc_lookup_v1_RouteLookupClusterSpecifier_route_lookup_config(specifier); if (plugin_config == nullptr) { - return absl::InvalidArgumentError( - "Could not get route lookup config from route lookup cluster " - "specifier"); + ValidationErrors::ScopedField field(errors, ".route_lookup_config"); + errors->AddError("field not present"); + return {}; } upb::Status status; const upb_MessageDef* msg_type = @@ -61,82 +83,54 @@ XdsRouteLookupClusterSpecifierPlugin::GenerateLoadBalancingPolicyConfig( size_t json_size = upb_JsonEncode(plugin_config, msg_type, symtab, 0, nullptr, 0, status.ptr()); if (json_size == static_cast<size_t>(-1)) { - return absl::InvalidArgumentError( - absl::StrCat("failed to dump proto to JSON: ", - upb_Status_ErrorMessage(status.ptr()))); + errors->AddError(absl::StrCat("failed to dump proto to JSON: ", + upb_Status_ErrorMessage(status.ptr()))); + return {}; } void* buf = upb_Arena_Malloc(arena, json_size + 1); upb_JsonEncode(plugin_config, msg_type, symtab, 0, reinterpret_cast<char*>(buf), json_size + 1, status.ptr()); - Json::Object rls_policy; - grpc_error_handle error = GRPC_ERROR_NONE; - rls_policy["routeLookupConfig"] = - Json::Parse(reinterpret_cast<char*>(buf), &error); - GPR_ASSERT(error == GRPC_ERROR_NONE); - Json::Object cds_policy; - cds_policy["cds_experimental"] = Json::Object(); - Json::Array child_policy; - child_policy.emplace_back(std::move(cds_policy)); - rls_policy["childPolicy"] = std::move(child_policy); - rls_policy["childPolicyConfigTargetFieldName"] = "cluster"; - Json::Object policy; - policy["rls_experimental"] = std::move(rls_policy); - Json::Array policies; - policies.emplace_back(std::move(policy)); - Json lb_policy_config(std::move(policies)); - grpc_error_handle parse_error = GRPC_ERROR_NONE; - // TODO(roth): If/when we ever add a second plugin, refactor this code - // somehow such that we automatically validate the resulting config against - // the gRPC LB policy registry instead of requiring each plugin to do that - // itself. - LoadBalancingPolicyRegistry::ParseLoadBalancingConfig(lb_policy_config, - &parse_error); - if (parse_error != GRPC_ERROR_NONE) { - absl::Status status = absl::InvalidArgumentError(absl::StrCat( - kXdsRouteLookupClusterSpecifierPluginConfigName, - " ClusterSpecifierPlugin returned invalid LB policy config: ", - grpc_error_std_string(parse_error))); - GRPC_ERROR_UNREF(parse_error); - return status; - } - return lb_policy_config.Dump(); + auto json = JsonParse(reinterpret_cast<char*>(buf)); + GPR_ASSERT(json.ok()); + return Json::FromArray({Json::FromObject( + {{"rls_experimental", + Json::FromObject({ + {"routeLookupConfig", std::move(*json)}, + {"childPolicy", + Json::FromArray({ + Json::FromObject({{"cds_experimental", Json::FromObject({})}}), + })}, + {"childPolicyConfigTargetFieldName", Json::FromString("cluster")}, + })}})}); } -namespace { - -using PluginRegistryMap = - std::map<absl::string_view, std::unique_ptr<XdsClusterSpecifierPluginImpl>>; +// +// XdsClusterSpecifierPluginRegistry +// -PluginRegistryMap* g_plugin_registry = nullptr; +XdsClusterSpecifierPluginRegistry::XdsClusterSpecifierPluginRegistry() { + RegisterPlugin(std::make_unique<XdsRouteLookupClusterSpecifierPlugin>()); +} -} // namespace +void XdsClusterSpecifierPluginRegistry::RegisterPlugin( + std::unique_ptr<XdsClusterSpecifierPluginImpl> plugin) { + absl::string_view name = plugin->ConfigProtoName(); + registry_[name] = std::move(plugin); +} const XdsClusterSpecifierPluginImpl* XdsClusterSpecifierPluginRegistry::GetPluginForType( - absl::string_view config_proto_type_name) { - auto it = g_plugin_registry->find(config_proto_type_name); - if (it == g_plugin_registry->end()) return nullptr; + absl::string_view config_proto_type_name) const { + auto it = registry_.find(config_proto_type_name); + if (it == registry_.end()) return nullptr; return it->second.get(); } -void XdsClusterSpecifierPluginRegistry::PopulateSymtab(upb_DefPool* symtab) { - for (const auto& p : *g_plugin_registry) { +void XdsClusterSpecifierPluginRegistry::PopulateSymtab( + upb_DefPool* symtab) const { + for (const auto& p : registry_) { p.second->PopulateSymtab(symtab); } } -void XdsClusterSpecifierPluginRegistry::RegisterPlugin( - std::unique_ptr<XdsClusterSpecifierPluginImpl> plugin, - absl::string_view config_proto_type_name) { - (*g_plugin_registry)[config_proto_type_name] = std::move(plugin); -} - -void XdsClusterSpecifierPluginRegistry::Init() { - g_plugin_registry = new PluginRegistryMap; - RegisterPlugin(absl::make_unique<XdsRouteLookupClusterSpecifierPlugin>(), - kXdsRouteLookupClusterSpecifierPluginConfigName); -} - -void XdsClusterSpecifierPluginRegistry::Shutdown() { delete g_plugin_registry; } - } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_cluster_specifier_plugin.h b/grpc/src/core/ext/xds/xds_cluster_specifier_plugin.h index 54513b02..bc2939ce 100644 --- a/grpc/src/core/ext/xds/xds_cluster_specifier_plugin.h +++ b/grpc/src/core/ext/xds/xds_cluster_specifier_plugin.h @@ -14,24 +14,21 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_CLUSTER_SPECIFIER_PLUGIN_H -#define GRPC_CORE_EXT_XDS_XDS_CLUSTER_SPECIFIER_PLUGIN_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_CLUSTER_SPECIFIER_PLUGIN_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_CLUSTER_SPECIFIER_PLUGIN_H #include <grpc/support/port_platform.h> +#include <map> #include <memory> -#include <set> -#include <string> +#include <utility> -#include "absl/status/statusor.h" -#include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "google/protobuf/any.upb.h" -#include "upb/def.h" +#include "upb/mem/arena.h" +#include "upb/reflection/def.h" -#include <grpc/grpc.h> - -#include "src/core/lib/channel/channel_stack.h" +#include "src/core/ext/xds/xds_common_types.h" +#include "src/core/lib/gprpp/validation_errors.h" #include "src/core/lib/json/json.h" namespace grpc_core { @@ -40,40 +37,61 @@ class XdsClusterSpecifierPluginImpl { public: virtual ~XdsClusterSpecifierPluginImpl() = default; + // Returns the config proto message name. + virtual absl::string_view ConfigProtoName() const = 0; + // Loads the proto message into the upb symtab. virtual void PopulateSymtab(upb_DefPool* symtab) const = 0; // Returns the LB policy config in JSON form. - virtual absl::StatusOr<std::string> GenerateLoadBalancingPolicyConfig( - upb_StringView serialized_plugin_config, upb_Arena* arena, - upb_DefPool* symtab) const = 0; + virtual Json GenerateLoadBalancingPolicyConfig( + XdsExtension extension, upb_Arena* arena, upb_DefPool* symtab, + ValidationErrors* errors) const = 0; }; class XdsRouteLookupClusterSpecifierPlugin : public XdsClusterSpecifierPluginImpl { + absl::string_view ConfigProtoName() const override; + void PopulateSymtab(upb_DefPool* symtab) const override; - absl::StatusOr<std::string> GenerateLoadBalancingPolicyConfig( - upb_StringView serialized_plugin_config, upb_Arena* arena, - upb_DefPool* symtab) const override; + Json GenerateLoadBalancingPolicyConfig( + XdsExtension extension, upb_Arena* arena, upb_DefPool* symtab, + ValidationErrors* errors) const override; }; class XdsClusterSpecifierPluginRegistry { public: - static void RegisterPlugin( - std::unique_ptr<XdsClusterSpecifierPluginImpl> plugin, - absl::string_view config_proto_type_name); - - static void PopulateSymtab(upb_DefPool* symtab); - - static const XdsClusterSpecifierPluginImpl* GetPluginForType( - absl::string_view config_proto_type_name); - - // Global init and shutdown. - static void Init(); - static void Shutdown(); + XdsClusterSpecifierPluginRegistry(); + + // Not copyable. + XdsClusterSpecifierPluginRegistry(const XdsClusterSpecifierPluginRegistry&) = + delete; + XdsClusterSpecifierPluginRegistry& operator=( + const XdsClusterSpecifierPluginRegistry&) = delete; + + // Movable. + XdsClusterSpecifierPluginRegistry( + XdsClusterSpecifierPluginRegistry&& other) noexcept + : registry_(std::move(other.registry_)) {} + XdsClusterSpecifierPluginRegistry& operator=( + XdsClusterSpecifierPluginRegistry&& other) noexcept { + registry_ = std::move(other.registry_); + return *this; + } + + void RegisterPlugin(std::unique_ptr<XdsClusterSpecifierPluginImpl> plugin); + + void PopulateSymtab(upb_DefPool* symtab) const; + + const XdsClusterSpecifierPluginImpl* GetPluginForType( + absl::string_view config_proto_type_name) const; + + private: + std::map<absl::string_view, std::unique_ptr<XdsClusterSpecifierPluginImpl>> + registry_; }; } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_CLUSTER_SPECIFIER_PLUGIN_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_CLUSTER_SPECIFIER_PLUGIN_H diff --git a/grpc/src/core/ext/xds/xds_common_types.cc b/grpc/src/core/ext/xds/xds_common_types.cc index 37083181..73985d17 100644 --- a/grpc/src/core/ext/xds/xds_common_types.cc +++ b/grpc/src/core/ext/xds/xds_common_types.cc @@ -18,7 +18,15 @@ #include "src/core/ext/xds/xds_common_types.h" -#include "absl/container/inlined_vector.h" +#include <stddef.h> +#include <stdint.h> + +#include <algorithm> +#include <initializer_list> +#include <map> +#include <utility> + +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" @@ -28,17 +36,50 @@ #include "envoy/type/matcher/v3/regex.upb.h" #include "envoy/type/matcher/v3/string.upb.h" #include "google/protobuf/any.upb.h" +#include "google/protobuf/struct.upb.h" +#include "google/protobuf/struct.upbdefs.h" #include "google/protobuf/wrappers.upb.h" +#include "upb/base/status.h" +#include "upb/json/encode.h" +#include "upb/mem/arena.h" +#include "upb/upb.hpp" #include "xds/type/v3/typed_struct.upb.h" +#include <grpc/support/json.h> + +#include "src/core/ext/xds/upb_utils.h" +#include "src/core/ext/xds/xds_bootstrap_grpc.h" +#include "src/core/ext/xds/xds_client.h" +#include "src/core/lib/json/json_reader.h" + namespace grpc_core { // +// ParseDuration() +// + +Duration ParseDuration(const google_protobuf_Duration* proto_duration, + ValidationErrors* errors) { + int64_t seconds = google_protobuf_Duration_seconds(proto_duration); + if (seconds < 0 || seconds > 315576000000) { + ValidationErrors::ScopedField field(errors, ".seconds"); + errors->AddError("value must be in the range [0, 315576000000]"); + } + int32_t nanos = google_protobuf_Duration_nanos(proto_duration); + if (nanos < 0 || nanos > 999999999) { + ValidationErrors::ScopedField field(errors, ".nanos"); + errors->AddError("value must be in the range [0, 999999999]"); + } + return Duration::FromSecondsAndNanoseconds(seconds, nanos); +} + +// // CommonTlsContext::CertificateValidationContext // std::string CommonTlsContext::CertificateValidationContext::ToString() const { std::vector<std::string> contents; + contents.reserve(match_subject_alt_names.size()); for (const auto& match : match_subject_alt_names) { contents.push_back(match.ToString()); } @@ -56,7 +97,7 @@ bool CommonTlsContext::CertificateValidationContext::Empty() const { std::string CommonTlsContext::CertificateProviderPluginInstance::ToString() const { - absl::InlinedVector<std::string, 2> contents; + std::vector<std::string> contents; if (!instance_name.empty()) { contents.push_back(absl::StrFormat("instance_name=%s", instance_name)); } @@ -76,7 +117,7 @@ bool CommonTlsContext::CertificateProviderPluginInstance::Empty() const { // std::string CommonTlsContext::ToString() const { - absl::InlinedVector<std::string, 2> contents; + std::vector<std::string> contents; if (!tls_certificate_provider_instance.Empty()) { contents.push_back( absl::StrFormat("tls_certificate_provider_instance=%s", @@ -102,64 +143,70 @@ namespace { // same CertificateProviderPluginInstance struct since the fields are the same. // TODO(yashykt): Remove this once we stop supporting the old way of fetching // certificate provider instances. -grpc_error_handle CertificateProviderInstanceParse( - const XdsEncodingContext& context, +CommonTlsContext::CertificateProviderPluginInstance +CertificateProviderInstanceParse( + const XdsResourceType::DecodeContext& context, const envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_CertificateProviderInstance* certificate_provider_instance_proto, - CommonTlsContext::CertificateProviderPluginInstance* - certificate_provider_plugin_instance) { - *certificate_provider_plugin_instance = { - UpbStringToStdString( - envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_CertificateProviderInstance_instance_name( - certificate_provider_instance_proto)), - UpbStringToStdString( - envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_CertificateProviderInstance_certificate_name( - certificate_provider_instance_proto))}; - if (context.certificate_provider_definition_map->find( - certificate_provider_plugin_instance->instance_name) == - context.certificate_provider_definition_map->end()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("Unrecognized certificate provider instance name: ", - certificate_provider_plugin_instance->instance_name)); + ValidationErrors* errors) { + CommonTlsContext::CertificateProviderPluginInstance cert_provider; + cert_provider.instance_name = UpbStringToStdString( + envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_CertificateProviderInstance_instance_name( + certificate_provider_instance_proto)); + const auto& bootstrap = + static_cast<const GrpcXdsBootstrap&>(context.client->bootstrap()); + if (bootstrap.certificate_providers().find(cert_provider.instance_name) == + bootstrap.certificate_providers().end()) { + ValidationErrors::ScopedField field(errors, ".instance_name"); + errors->AddError( + absl::StrCat("unrecognized certificate provider instance name: ", + cert_provider.instance_name)); } - return GRPC_ERROR_NONE; + cert_provider.certificate_name = UpbStringToStdString( + envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_CertificateProviderInstance_certificate_name( + certificate_provider_instance_proto)); + return cert_provider; } -grpc_error_handle CertificateProviderPluginInstanceParse( - const XdsEncodingContext& context, +CommonTlsContext::CertificateProviderPluginInstance +CertificateProviderPluginInstanceParse( + const XdsResourceType::DecodeContext& context, const envoy_extensions_transport_sockets_tls_v3_CertificateProviderPluginInstance* certificate_provider_plugin_instance_proto, - CommonTlsContext::CertificateProviderPluginInstance* - certificate_provider_plugin_instance) { - *certificate_provider_plugin_instance = { - UpbStringToStdString( - envoy_extensions_transport_sockets_tls_v3_CertificateProviderPluginInstance_instance_name( - certificate_provider_plugin_instance_proto)), - UpbStringToStdString( - envoy_extensions_transport_sockets_tls_v3_CertificateProviderPluginInstance_certificate_name( - certificate_provider_plugin_instance_proto))}; - if (context.certificate_provider_definition_map->find( - certificate_provider_plugin_instance->instance_name) == - context.certificate_provider_definition_map->end()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("Unrecognized certificate provider instance name: ", - certificate_provider_plugin_instance->instance_name)); + ValidationErrors* errors) { + CommonTlsContext::CertificateProviderPluginInstance cert_provider; + cert_provider.instance_name = UpbStringToStdString( + envoy_extensions_transport_sockets_tls_v3_CertificateProviderPluginInstance_instance_name( + certificate_provider_plugin_instance_proto)); + const auto& bootstrap = + static_cast<const GrpcXdsBootstrap&>(context.client->bootstrap()); + if (bootstrap.certificate_providers().find(cert_provider.instance_name) == + bootstrap.certificate_providers().end()) { + ValidationErrors::ScopedField field(errors, ".instance_name"); + errors->AddError( + absl::StrCat("unrecognized certificate provider instance name: ", + cert_provider.instance_name)); } - return GRPC_ERROR_NONE; + cert_provider.certificate_name = UpbStringToStdString( + envoy_extensions_transport_sockets_tls_v3_CertificateProviderPluginInstance_certificate_name( + certificate_provider_plugin_instance_proto)); + return cert_provider; } -grpc_error_handle CertificateValidationContextParse( - const XdsEncodingContext& context, +CommonTlsContext::CertificateValidationContext +CertificateValidationContextParse( + const XdsResourceType::DecodeContext& context, const envoy_extensions_transport_sockets_tls_v3_CertificateValidationContext* certificate_validation_context_proto, - CommonTlsContext::CertificateValidationContext* - certificate_validation_context) { - std::vector<grpc_error_handle> errors; + ValidationErrors* errors) { + CommonTlsContext::CertificateValidationContext certificate_validation_context; size_t len = 0; auto* subject_alt_names_matchers = envoy_extensions_transport_sockets_tls_v3_CertificateValidationContext_match_subject_alt_names( certificate_validation_context_proto, &len); for (size_t i = 0; i < len; ++i) { + ValidationErrors::ScopedField field( + errors, absl::StrCat(".match_subject_alt_names[", i, "]")); StringMatcher::Type type; std::string matcher; if (envoy_type_matcher_v3_StringMatcher_has_exact( @@ -191,8 +238,7 @@ grpc_error_handle CertificateValidationContextParse( matcher = UpbStringToStdString( envoy_type_matcher_v3_RegexMatcher_regex(regex_matcher)); } else { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Invalid StringMatcher specified")); + errors->AddError("invalid StringMatcher specified"); continue; } bool ignore_case = envoy_type_matcher_v3_StringMatcher_ignore_case( @@ -201,71 +247,67 @@ grpc_error_handle CertificateValidationContextParse( StringMatcher::Create(type, matcher, /*case_sensitive=*/!ignore_case); if (!string_matcher.ok()) { - errors.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("string matcher: ", string_matcher.status().message()))); + errors->AddError(string_matcher.status().message()); continue; } if (type == StringMatcher::Type::kSafeRegex && ignore_case) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "StringMatcher: ignore_case has no effect for SAFE_REGEX.")); + ValidationErrors::ScopedField field(errors, ".ignore_case"); + errors->AddError("not supported for regex matcher"); continue; } - certificate_validation_context->match_subject_alt_names.push_back( + certificate_validation_context.match_subject_alt_names.push_back( std::move(string_matcher.value())); } auto* ca_certificate_provider_instance = envoy_extensions_transport_sockets_tls_v3_CertificateValidationContext_ca_certificate_provider_instance( certificate_validation_context_proto); if (ca_certificate_provider_instance != nullptr) { - grpc_error_handle error = CertificateProviderPluginInstanceParse( - context, ca_certificate_provider_instance, - &certificate_validation_context->ca_certificate_provider_instance); - if (error != GRPC_ERROR_NONE) errors.push_back(error); + ValidationErrors::ScopedField field(errors, + ".ca_certificate_provider_instance"); + certificate_validation_context.ca_certificate_provider_instance = + CertificateProviderPluginInstanceParse( + context, ca_certificate_provider_instance, errors); } if (envoy_extensions_transport_sockets_tls_v3_CertificateValidationContext_verify_certificate_spki( certificate_validation_context_proto, nullptr) != nullptr) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "CertificateValidationContext: verify_certificate_spki " - "unsupported")); + ValidationErrors::ScopedField field(errors, ".verify_certificate_spki"); + errors->AddError("feature unsupported"); } if (envoy_extensions_transport_sockets_tls_v3_CertificateValidationContext_verify_certificate_hash( certificate_validation_context_proto, nullptr) != nullptr) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "CertificateValidationContext: verify_certificate_hash " - "unsupported")); + ValidationErrors::ScopedField field(errors, ".verify_certificate_hash"); + errors->AddError("feature unsupported"); } auto* require_signed_certificate_timestamp = envoy_extensions_transport_sockets_tls_v3_CertificateValidationContext_require_signed_certificate_timestamp( certificate_validation_context_proto); if (require_signed_certificate_timestamp != nullptr && google_protobuf_BoolValue_value(require_signed_certificate_timestamp)) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "CertificateValidationContext: " - "require_signed_certificate_timestamp unsupported")); + ValidationErrors::ScopedField field( + errors, ".require_signed_certificate_timestamp"); + errors->AddError("feature unsupported"); } if (envoy_extensions_transport_sockets_tls_v3_CertificateValidationContext_has_crl( certificate_validation_context_proto)) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "CertificateValidationContext: crl unsupported")); + ValidationErrors::ScopedField field(errors, ".crl"); + errors->AddError("feature unsupported"); } if (envoy_extensions_transport_sockets_tls_v3_CertificateValidationContext_has_custom_validator_config( certificate_validation_context_proto)) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "CertificateValidationContext: custom_validator_config " - "unsupported")); + ValidationErrors::ScopedField field(errors, ".custom_validator_config"); + errors->AddError("feature unsupported"); } - return GRPC_ERROR_CREATE_FROM_VECTOR( - "Error parsing CertificateValidationContext", &errors); + return certificate_validation_context; } } // namespace -grpc_error_handle CommonTlsContext::Parse( - const XdsEncodingContext& context, +CommonTlsContext CommonTlsContext::Parse( + const XdsResourceType::DecodeContext& context, const envoy_extensions_transport_sockets_tls_v3_CommonTlsContext* common_tls_context_proto, - CommonTlsContext* common_tls_context) { - std::vector<grpc_error_handle> errors; + ValidationErrors* errors) { + CommonTlsContext common_tls_context; // The validation context is derived from the oneof in // 'validation_context_type'. 'validation_context_sds_secret_config' is not // supported. @@ -273,14 +315,16 @@ grpc_error_handle CommonTlsContext::Parse( envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_combined_validation_context( common_tls_context_proto); if (combined_validation_context != nullptr) { + ValidationErrors::ScopedField field(errors, ".combined_validation_context"); auto* default_validation_context = envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_CombinedCertificateValidationContext_default_validation_context( combined_validation_context); if (default_validation_context != nullptr) { - grpc_error_handle error = CertificateValidationContextParse( - context, default_validation_context, - &common_tls_context->certificate_validation_context); - if (error != GRPC_ERROR_NONE) errors.push_back(error); + ValidationErrors::ScopedField field(errors, + ".default_validation_context"); + common_tls_context.certificate_validation_context = + CertificateValidationContextParse(context, default_validation_context, + errors); } // If after parsing default_validation_context, // common_tls_context->certificate_validation_context.ca_certificate_provider_instance @@ -289,42 +333,44 @@ grpc_error_handle CommonTlsContext::Parse( // 'combined_validation_context'. Note that this way of fetching root // certificates is deprecated and will be removed in the future. // TODO(yashykt): Remove this once it's no longer needed. - auto* validation_context_certificate_provider_instance = + const auto* validation_context_certificate_provider_instance = envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_CombinedCertificateValidationContext_validation_context_certificate_provider_instance( combined_validation_context); - if (common_tls_context->certificate_validation_context + if (common_tls_context.certificate_validation_context .ca_certificate_provider_instance.Empty() && validation_context_certificate_provider_instance != nullptr) { - grpc_error_handle error = CertificateProviderInstanceParse( - context, validation_context_certificate_provider_instance, - &common_tls_context->certificate_validation_context - .ca_certificate_provider_instance); - if (error != GRPC_ERROR_NONE) errors.push_back(error); + ValidationErrors::ScopedField field( + errors, ".validation_context_certificate_provider_instance"); + common_tls_context.certificate_validation_context + .ca_certificate_provider_instance = CertificateProviderInstanceParse( + context, validation_context_certificate_provider_instance, errors); } } else { auto* validation_context = envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_validation_context( common_tls_context_proto); if (validation_context != nullptr) { - grpc_error_handle error = CertificateValidationContextParse( - context, validation_context, - &common_tls_context->certificate_validation_context); - if (error != GRPC_ERROR_NONE) errors.push_back(error); + ValidationErrors::ScopedField field(errors, ".validation_context"); + common_tls_context.certificate_validation_context = + CertificateValidationContextParse(context, validation_context, + errors); } else if ( envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_has_validation_context_sds_secret_config( common_tls_context_proto)) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "validation_context_sds_secret_config unsupported")); + ValidationErrors::ScopedField field( + errors, ".validation_context_sds_secret_config"); + errors->AddError("feature unsupported"); } } auto* tls_certificate_provider_instance = envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_tls_certificate_provider_instance( common_tls_context_proto); if (tls_certificate_provider_instance != nullptr) { - grpc_error_handle error = CertificateProviderPluginInstanceParse( - context, tls_certificate_provider_instance, - &common_tls_context->tls_certificate_provider_instance); - if (error != GRPC_ERROR_NONE) errors.push_back(error); + ValidationErrors::ScopedField field(errors, + ".tls_certificate_provider_instance"); + common_tls_context.tls_certificate_provider_instance = + CertificateProviderPluginInstanceParse( + context, tls_certificate_provider_instance, errors); } else { // Fall back onto 'tls_certificate_certificate_provider_instance'. Note that // this way of fetching identity certificates is deprecated and will be @@ -334,55 +380,127 @@ grpc_error_handle CommonTlsContext::Parse( envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_tls_certificate_certificate_provider_instance( common_tls_context_proto); if (tls_certificate_certificate_provider_instance != nullptr) { - grpc_error_handle error = CertificateProviderInstanceParse( - context, tls_certificate_certificate_provider_instance, - &common_tls_context->tls_certificate_provider_instance); - if (error != GRPC_ERROR_NONE) errors.push_back(error); + ValidationErrors::ScopedField field( + errors, ".tls_certificate_certificate_provider_instance"); + common_tls_context.tls_certificate_provider_instance = + CertificateProviderInstanceParse( + context, tls_certificate_certificate_provider_instance, errors); } else { if (envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_has_tls_certificates( common_tls_context_proto)) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "tls_certificates unsupported")); + ValidationErrors::ScopedField field(errors, ".tls_certificates"); + errors->AddError("feature unsupported"); } if (envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_has_tls_certificate_sds_secret_configs( common_tls_context_proto)) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "tls_certificate_sds_secret_configs unsupported")); + ValidationErrors::ScopedField field( + errors, ".tls_certificate_sds_secret_configs"); + errors->AddError("feature unsupported"); } } } if (envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_has_tls_params( common_tls_context_proto)) { - errors.push_back( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("tls_params unsupported")); + ValidationErrors::ScopedField field(errors, ".tls_params"); + errors->AddError("feature unsupported"); } if (envoy_extensions_transport_sockets_tls_v3_CommonTlsContext_has_custom_handshaker( common_tls_context_proto)) { - errors.push_back( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("custom_handshaker unsupported")); + ValidationErrors::ScopedField field(errors, ".custom_handshaker"); + errors->AddError("feature unsupported"); + } + return common_tls_context; +} + +// +// ExtractXdsExtension +// + +namespace { + +absl::StatusOr<Json> ParseProtobufStructToJson( + const XdsResourceType::DecodeContext& context, + const google_protobuf_Struct* resource) { + upb::Status status; + const auto* msg_def = google_protobuf_Struct_getmsgdef(context.symtab); + size_t json_size = upb_JsonEncode(resource, msg_def, context.symtab, 0, + nullptr, 0, status.ptr()); + if (json_size == static_cast<size_t>(-1)) { + return absl::InvalidArgumentError( + absl::StrCat("error encoding google::Protobuf::Struct as JSON: ", + upb_Status_ErrorMessage(status.ptr()))); + } + void* buf = upb_Arena_Malloc(context.arena, json_size + 1); + upb_JsonEncode(resource, msg_def, context.symtab, 0, + reinterpret_cast<char*>(buf), json_size + 1, status.ptr()); + auto json = JsonParse(reinterpret_cast<char*>(buf)); + if (!json.ok()) { + // This should never happen. + return absl::InternalError( + absl::StrCat("error parsing JSON form of google::Protobuf::Struct " + "produced by upb library: ", + json.status().ToString())); } - return GRPC_ERROR_CREATE_FROM_VECTOR("Error parsing CommonTlsContext", - &errors); + return std::move(*json); } -grpc_error_handle ExtractExtensionTypeName(const XdsEncodingContext& context, - const google_protobuf_Any* any, - absl::string_view* extension_type) { - *extension_type = UpbStringToAbsl(google_protobuf_Any_type_url(any)); - if (*extension_type == "type.googleapis.com/xds.type.v3.TypedStruct" || - *extension_type == "type.googleapis.com/udpa.type.v1.TypedStruct") { - upb_StringView any_value = google_protobuf_Any_value(any); +} // namespace + +absl::optional<XdsExtension> ExtractXdsExtension( + const XdsResourceType::DecodeContext& context, + const google_protobuf_Any* any, ValidationErrors* errors) { + if (any == nullptr) { + errors->AddError("field not present"); + return absl::nullopt; + } + XdsExtension extension; + auto strip_type_prefix = [&]() { + ValidationErrors::ScopedField field(errors, ".type_url"); + if (extension.type.empty()) { + errors->AddError("field not present"); + return false; + } + size_t pos = extension.type.rfind('/'); + if (pos == absl::string_view::npos || pos == extension.type.size() - 1) { + errors->AddError(absl::StrCat("invalid value \"", extension.type, "\"")); + } else { + extension.type = extension.type.substr(pos + 1); + } + return true; + }; + extension.type = UpbStringToAbsl(google_protobuf_Any_type_url(any)); + if (!strip_type_prefix()) return absl::nullopt; + extension.validation_fields.emplace_back( + errors, absl::StrCat(".value[", extension.type, "]")); + absl::string_view any_value = UpbStringToAbsl(google_protobuf_Any_value(any)); + if (extension.type == "xds.type.v3.TypedStruct" || + extension.type == "udpa.type.v1.TypedStruct") { const auto* typed_struct = xds_type_v3_TypedStruct_parse( - any_value.data, any_value.size, context.arena); + any_value.data(), any_value.size(), context.arena); if (typed_struct == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "could not parse TypedStruct from extension"); + errors->AddError("could not parse"); + return absl::nullopt; } - *extension_type = + extension.type = UpbStringToAbsl(xds_type_v3_TypedStruct_type_url(typed_struct)); + if (!strip_type_prefix()) return absl::nullopt; + extension.validation_fields.emplace_back( + errors, absl::StrCat(".value[", extension.type, "]")); + auto* protobuf_struct = xds_type_v3_TypedStruct_value(typed_struct); + if (protobuf_struct == nullptr) { + extension.value = Json::FromObject({}); // Default to empty object. + } else { + auto json = ParseProtobufStructToJson(context, protobuf_struct); + if (!json.ok()) { + errors->AddError(json.status().message()); + return absl::nullopt; + } + extension.value = std::move(*json); + } + } else { + extension.value = any_value; } - *extension_type = absl::StripPrefix(*extension_type, "type.googleapis.com/"); - return GRPC_ERROR_NONE; + return std::move(extension); } } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_common_types.h b/grpc/src/core/ext/xds/xds_common_types.h index c5e118bd..4ddc1b8a 100644 --- a/grpc/src/core/ext/xds/xds_common_types.h +++ b/grpc/src/core/ext/xds/xds_common_types.h @@ -14,29 +14,31 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_COMMON_TYPES_H -#define GRPC_CORE_EXT_XDS_XDS_COMMON_TYPES_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_COMMON_TYPES_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_COMMON_TYPES_H #include <grpc/support/port_platform.h> #include <string> #include <vector> -#include "absl/strings/str_format.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/variant.h" #include "envoy/extensions/transport_sockets/tls/v3/tls.upb.h" #include "google/protobuf/any.upb.h" #include "google/protobuf/duration.upb.h" -#include "src/core/ext/xds/upb_utils.h" +#include "src/core/ext/xds/xds_resource_type.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/gprpp/validation_errors.h" +#include "src/core/lib/json/json.h" #include "src/core/lib/matchers/matchers.h" namespace grpc_core { -inline Duration ParseDuration(const google_protobuf_Duration* proto_duration) { - return Duration::FromSecondsAndNanoseconds( - google_protobuf_Duration_seconds(proto_duration), - google_protobuf_Duration_nanos(proto_duration)); -} +Duration ParseDuration(const google_protobuf_Duration* proto_duration, + ValidationErrors* errors); struct CommonTlsContext { struct CertificateProviderPluginInstance { @@ -79,17 +81,28 @@ struct CommonTlsContext { std::string ToString() const; bool Empty() const; - static grpc_error_handle Parse( - const XdsEncodingContext& context, + static CommonTlsContext Parse( + const XdsResourceType::DecodeContext& context, const envoy_extensions_transport_sockets_tls_v3_CommonTlsContext* common_tls_context_proto, - CommonTlsContext* common_tls_context); + ValidationErrors* errors); }; -grpc_error_handle ExtractExtensionTypeName(const XdsEncodingContext& context, - const google_protobuf_Any* any, - absl::string_view* extension_type); +struct XdsExtension { + // The type, either from the top level or from inside the TypedStruct. + absl::string_view type; + // A Json object for a TypedStruct, or the serialized config otherwise. + absl::variant<absl::string_view /*serialized_value*/, Json /*typed_struct*/> + value; + // Validation fields that need to stay in scope until we're done + // processing the extension. + std::vector<ValidationErrors::ScopedField> validation_fields; +}; + +absl::optional<XdsExtension> ExtractXdsExtension( + const XdsResourceType::DecodeContext& context, + const google_protobuf_Any* any, ValidationErrors* errors); } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_COMMON_TYPES_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_COMMON_TYPES_H diff --git a/grpc/src/core/ext/xds/xds_endpoint.cc b/grpc/src/core/ext/xds/xds_endpoint.cc index 89f433f6..2638423d 100644 --- a/grpc/src/core/ext/xds/xds_endpoint.cc +++ b/grpc/src/core/ext/xds/xds_endpoint.cc @@ -18,9 +18,19 @@ #include "src/core/ext/xds/xds_endpoint.h" -#include "absl/memory/memory.h" +#include <stdlib.h> +#include <string.h> + +#include <algorithm> +#include <limits> +#include <set> +#include <vector> + +#include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" +#include "absl/types/optional.h" #include "envoy/config/core/v3/address.upb.h" #include "envoy/config/core/v3/base.upb.h" #include "envoy/config/core/v3/health_check.upb.h" @@ -29,13 +39,20 @@ #include "envoy/config/endpoint/v3/endpoint_components.upb.h" #include "envoy/type/v3/percent.upb.h" #include "google/protobuf/wrappers.upb.h" -#include "upb/text_encode.h" -#include "upb/upb.h" -#include "upb/upb.hpp" +#include "upb/text/encode.h" + +#include <grpc/support/log.h> #include "src/core/ext/xds/upb_utils.h" +#include "src/core/ext/xds/xds_cluster.h" +#include "src/core/ext/xds/xds_health_status.h" +#include "src/core/ext/xds/xds_resource_type.h" #include "src/core/lib/address_utils/parse_address.h" #include "src/core/lib/address_utils/sockaddr_utils.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/validation_errors.h" +#include "src/core/lib/iomgr/resolved_address.h" namespace grpc_core { @@ -68,6 +85,7 @@ bool XdsEndpointResource::Priority::operator==(const Priority& other) const { std::string XdsEndpointResource::Priority::ToString() const { std::vector<std::string> locality_strings; + locality_strings.reserve(localities.size()); for (const auto& p : localities) { locality_strings.emplace_back(p.second.ToString()); } @@ -75,11 +93,14 @@ std::string XdsEndpointResource::Priority::ToString() const { } bool XdsEndpointResource::DropConfig::ShouldDrop( - const std::string** category_name) const { + const std::string** category_name) { for (size_t i = 0; i < drop_category_list_.size(); ++i) { const auto& drop_category = drop_category_list_[i]; // Generate a random number in [0, 1000000). - const uint32_t random = static_cast<uint32_t>(rand()) % 1000000; + const uint32_t random = [&]() { + MutexLock lock(&mu_); + return absl::Uniform<uint32_t>(bit_gen_, 0, 1000000); + }(); if (random < drop_category.parts_per_million) { *category_name = &drop_category.name; return true; @@ -116,7 +137,7 @@ std::string XdsEndpointResource::ToString() const { namespace { void MaybeLogClusterLoadAssignment( - const XdsEncodingContext& context, + const XdsResourceType::DecodeContext& context, const envoy_config_endpoint_v3_ClusterLoadAssignment* cla) { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { @@ -130,242 +151,330 @@ void MaybeLogClusterLoadAssignment( } } -grpc_error_handle ServerAddressParseAndAppend( +absl::optional<ServerAddress> ServerAddressParse( const envoy_config_endpoint_v3_LbEndpoint* lb_endpoint, - ServerAddressList* list) { - // If health_status is not HEALTHY or UNKNOWN, skip this endpoint. + ValidationErrors* errors) { + // health_status const int32_t health_status = envoy_config_endpoint_v3_LbEndpoint_health_status(lb_endpoint); - if (health_status != envoy_config_core_v3_UNKNOWN && + if (!XdsOverrideHostEnabled() && + health_status != envoy_config_core_v3_UNKNOWN && health_status != envoy_config_core_v3_HEALTHY) { - return GRPC_ERROR_NONE; + return absl::nullopt; } - // Find the ip:port. - const envoy_config_endpoint_v3_Endpoint* endpoint = - envoy_config_endpoint_v3_LbEndpoint_endpoint(lb_endpoint); - const envoy_config_core_v3_Address* address = - envoy_config_endpoint_v3_Endpoint_address(endpoint); - const envoy_config_core_v3_SocketAddress* socket_address = - envoy_config_core_v3_Address_socket_address(address); - std::string address_str = UpbStringToStdString( - envoy_config_core_v3_SocketAddress_address(socket_address)); - uint32_t port = envoy_config_core_v3_SocketAddress_port_value(socket_address); - if (GPR_UNLIKELY(port >> 16) != 0) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Invalid port."); + auto status = XdsHealthStatus::FromUpb(health_status); + if (!status.has_value()) return absl::nullopt; + // load_balancing_weight + uint32_t weight = 1; + { + ValidationErrors::ScopedField field(errors, ".load_balancing_weight"); + const google_protobuf_UInt32Value* load_balancing_weight = + envoy_config_endpoint_v3_LbEndpoint_load_balancing_weight(lb_endpoint); + if (load_balancing_weight != nullptr) { + weight = google_protobuf_UInt32Value_value(load_balancing_weight); + if (weight == 0) { + errors->AddError("must be greater than 0"); + } + } } - // Find load_balancing_weight for the endpoint. - const google_protobuf_UInt32Value* load_balancing_weight = - envoy_config_endpoint_v3_LbEndpoint_load_balancing_weight(lb_endpoint); - const int32_t weight = - load_balancing_weight != nullptr - ? google_protobuf_UInt32Value_value(load_balancing_weight) - : 500; - if (weight == 0) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Invalid endpoint weight of 0."); + // endpoint + grpc_resolved_address grpc_address; + { + ValidationErrors::ScopedField field(errors, ".endpoint"); + const envoy_config_endpoint_v3_Endpoint* endpoint = + envoy_config_endpoint_v3_LbEndpoint_endpoint(lb_endpoint); + if (endpoint == nullptr) { + errors->AddError("field not present"); + return absl::nullopt; + } + ValidationErrors::ScopedField field2(errors, ".address"); + const envoy_config_core_v3_Address* address = + envoy_config_endpoint_v3_Endpoint_address(endpoint); + if (address == nullptr) { + errors->AddError("field not present"); + return absl::nullopt; + } + ValidationErrors::ScopedField field3(errors, ".socket_address"); + const envoy_config_core_v3_SocketAddress* socket_address = + envoy_config_core_v3_Address_socket_address(address); + if (socket_address == nullptr) { + errors->AddError("field not present"); + return absl::nullopt; + } + std::string address_str = UpbStringToStdString( + envoy_config_core_v3_SocketAddress_address(socket_address)); + uint32_t port; + { + ValidationErrors::ScopedField field(errors, ".port_value"); + port = envoy_config_core_v3_SocketAddress_port_value(socket_address); + if (GPR_UNLIKELY(port >> 16) != 0) { + errors->AddError("invalid port"); + return absl::nullopt; + } + } + auto addr = StringToSockaddr(address_str, port); + if (!addr.ok()) { + errors->AddError(addr.status().message()); + } else { + grpc_address = *addr; + } } - // Populate grpc_resolved_address. - grpc_resolved_address addr; - grpc_error_handle error = - grpc_string_to_sockaddr(&addr, address_str.c_str(), port); - if (error != GRPC_ERROR_NONE) return error; - // Append the address to the list. + // Convert to ServerAddress. std::map<const char*, std::unique_ptr<ServerAddress::AttributeInterface>> attributes; attributes[ServerAddressWeightAttribute::kServerAddressWeightAttributeKey] = - absl::make_unique<ServerAddressWeightAttribute>(weight); - list->emplace_back(addr, nullptr, std::move(attributes)); - return GRPC_ERROR_NONE; + std::make_unique<ServerAddressWeightAttribute>(weight); + attributes[XdsEndpointHealthStatusAttribute::kKey] = + std::make_unique<XdsEndpointHealthStatusAttribute>(*status); + return ServerAddress(grpc_address, ChannelArgs(), std::move(attributes)); } -grpc_error_handle LocalityParse( +struct ParsedLocality { + size_t priority; + XdsEndpointResource::Priority::Locality locality; +}; + +struct ResolvedAddressLessThan { + bool operator()(const grpc_resolved_address& a1, + const grpc_resolved_address& a2) const { + if (a1.len != a2.len) return a1.len < a2.len; + return memcmp(a1.addr, a2.addr, a1.len) < 0; + } +}; +using ResolvedAddressSet = + std::set<grpc_resolved_address, ResolvedAddressLessThan>; + +absl::optional<ParsedLocality> LocalityParse( const envoy_config_endpoint_v3_LocalityLbEndpoints* locality_lb_endpoints, - XdsEndpointResource::Priority::Locality* output_locality, - size_t* priority) { - // Parse LB weight. + ResolvedAddressSet* address_set, ValidationErrors* errors) { + const size_t original_error_size = errors->size(); + ParsedLocality parsed_locality; + // load_balancing_weight + // If LB weight is not specified or 0, it means this locality is assigned + // no load. const google_protobuf_UInt32Value* lb_weight = envoy_config_endpoint_v3_LocalityLbEndpoints_load_balancing_weight( locality_lb_endpoints); - // If LB weight is not specified, it means this locality is assigned no load. - // TODO(juanlishen): When we support CDS to configure the inter-locality - // policy, we should change the LB weight handling. - output_locality->lb_weight = + parsed_locality.locality.lb_weight = lb_weight != nullptr ? google_protobuf_UInt32Value_value(lb_weight) : 0; - if (output_locality->lb_weight == 0) return GRPC_ERROR_NONE; - // Parse locality name. + if (parsed_locality.locality.lb_weight == 0) return absl::nullopt; + // locality const envoy_config_core_v3_Locality* locality = envoy_config_endpoint_v3_LocalityLbEndpoints_locality( locality_lb_endpoints); if (locality == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty locality."); + ValidationErrors::ScopedField field(errors, ".locality"); + errors->AddError("field not present"); + return absl::nullopt; } + // region std::string region = UpbStringToStdString(envoy_config_core_v3_Locality_region(locality)); + // zone std::string zone = UpbStringToStdString(envoy_config_core_v3_Locality_zone(locality)); + // sub_zone std::string sub_zone = UpbStringToStdString(envoy_config_core_v3_Locality_sub_zone(locality)); - output_locality->name = MakeRefCounted<XdsLocalityName>( + parsed_locality.locality.name = MakeRefCounted<XdsLocalityName>( std::move(region), std::move(zone), std::move(sub_zone)); - // Parse the addresses. + // lb_endpoints size_t size; const envoy_config_endpoint_v3_LbEndpoint* const* lb_endpoints = envoy_config_endpoint_v3_LocalityLbEndpoints_lb_endpoints( locality_lb_endpoints, &size); for (size_t i = 0; i < size; ++i) { - grpc_error_handle error = ServerAddressParseAndAppend( - lb_endpoints[i], &output_locality->endpoints); - if (error != GRPC_ERROR_NONE) return error; + ValidationErrors::ScopedField field(errors, + absl::StrCat(".lb_endpoints[", i, "]")); + auto address = ServerAddressParse(lb_endpoints[i], errors); + if (address.has_value()) { + bool inserted = address_set->insert(address->address()).second; + if (!inserted) { + errors->AddError(absl::StrCat( + "duplicate endpoint address \"", + grpc_sockaddr_to_uri(&address->address()).value_or("<unknown>"), + "\"")); + } + parsed_locality.locality.endpoints.push_back(std::move(*address)); + } } - // Parse the priority. - *priority = envoy_config_endpoint_v3_LocalityLbEndpoints_priority( - locality_lb_endpoints); - return GRPC_ERROR_NONE; + // priority + parsed_locality.priority = + envoy_config_endpoint_v3_LocalityLbEndpoints_priority( + locality_lb_endpoints); + // Return result. + if (original_error_size != errors->size()) return absl::nullopt; + return parsed_locality; } -grpc_error_handle DropParseAndAppend( +void DropParseAndAppend( const envoy_config_endpoint_v3_ClusterLoadAssignment_Policy_DropOverload* drop_overload, - XdsEndpointResource::DropConfig* drop_config) { - // Get the category. + XdsEndpointResource::DropConfig* drop_config, ValidationErrors* errors) { + // category std::string category = UpbStringToStdString( envoy_config_endpoint_v3_ClusterLoadAssignment_Policy_DropOverload_category( drop_overload)); if (category.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Empty drop category name"); + ValidationErrors::ScopedField field(errors, ".category"); + errors->AddError("empty drop category name"); } - // Get the drop rate (per million). - const envoy_type_v3_FractionalPercent* drop_percentage = - envoy_config_endpoint_v3_ClusterLoadAssignment_Policy_DropOverload_drop_percentage( - drop_overload); - uint32_t numerator = - envoy_type_v3_FractionalPercent_numerator(drop_percentage); - const auto denominator = - static_cast<envoy_type_v3_FractionalPercent_DenominatorType>( - envoy_type_v3_FractionalPercent_denominator(drop_percentage)); - // Normalize to million. - switch (denominator) { - case envoy_type_v3_FractionalPercent_HUNDRED: - numerator *= 10000; - break; - case envoy_type_v3_FractionalPercent_TEN_THOUSAND: - numerator *= 100; - break; - case envoy_type_v3_FractionalPercent_MILLION: - break; - default: - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Unknown denominator type"); + // drop_percentage + uint32_t numerator; + { + ValidationErrors::ScopedField field(errors, ".drop_percentage"); + const envoy_type_v3_FractionalPercent* drop_percentage = + envoy_config_endpoint_v3_ClusterLoadAssignment_Policy_DropOverload_drop_percentage( + drop_overload); + if (drop_percentage == nullptr) { + errors->AddError("field not present"); + return; + } + numerator = envoy_type_v3_FractionalPercent_numerator(drop_percentage); + { + ValidationErrors::ScopedField field(errors, ".denominator"); + const int denominator = + envoy_type_v3_FractionalPercent_denominator(drop_percentage); + // Normalize to million. + switch (denominator) { + case envoy_type_v3_FractionalPercent_HUNDRED: + numerator *= 10000; + break; + case envoy_type_v3_FractionalPercent_TEN_THOUSAND: + numerator *= 100; + break; + case envoy_type_v3_FractionalPercent_MILLION: + break; + default: + errors->AddError("unknown denominator type"); + } + } + // Cap numerator to 1000000. + numerator = std::min(numerator, 1000000u); } - // Cap numerator to 1000000. - numerator = std::min(numerator, 1000000u); + // Add category. drop_config->AddCategory(std::move(category), numerator); - return GRPC_ERROR_NONE; } -grpc_error_handle EdsResourceParse( - const XdsEncodingContext& /*context*/, +absl::StatusOr<XdsEndpointResource> EdsResourceParse( + const XdsResourceType::DecodeContext& /*context*/, const envoy_config_endpoint_v3_ClusterLoadAssignment* - cluster_load_assignment, - bool /*is_v2*/, XdsEndpointResource* eds_update) { - std::vector<grpc_error_handle> errors; - // Get the endpoints. - size_t locality_size; - const envoy_config_endpoint_v3_LocalityLbEndpoints* const* endpoints = - envoy_config_endpoint_v3_ClusterLoadAssignment_endpoints( - cluster_load_assignment, &locality_size); - for (size_t j = 0; j < locality_size; ++j) { - size_t priority; - XdsEndpointResource::Priority::Locality locality; - grpc_error_handle error = LocalityParse(endpoints[j], &locality, &priority); - if (error != GRPC_ERROR_NONE) { - errors.push_back(error); - continue; - } - // Filter out locality with weight 0. - if (locality.lb_weight == 0) continue; - // Make sure prorities is big enough. Note that they might not - // arrive in priority order. - if (eds_update->priorities.size() < priority + 1) { - eds_update->priorities.resize(priority + 1); - } - auto& locality_map = eds_update->priorities[priority].localities; - auto it = locality_map.find(locality.name.get()); - if (it != locality_map.end()) { - errors.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat( - "duplicate locality ", locality.name->AsHumanReadableString(), - " found in priority ", priority))); - } else { - locality_map.emplace(locality.name.get(), std::move(locality)); + cluster_load_assignment) { + ValidationErrors errors; + XdsEndpointResource eds_resource; + // endpoints + { + ValidationErrors::ScopedField field(&errors, "endpoints"); + ResolvedAddressSet address_set; + size_t locality_size; + const envoy_config_endpoint_v3_LocalityLbEndpoints* const* endpoints = + envoy_config_endpoint_v3_ClusterLoadAssignment_endpoints( + cluster_load_assignment, &locality_size); + for (size_t i = 0; i < locality_size; ++i) { + ValidationErrors::ScopedField field(&errors, absl::StrCat("[", i, "]")); + auto parsed_locality = LocalityParse(endpoints[i], &address_set, &errors); + if (parsed_locality.has_value()) { + GPR_ASSERT(parsed_locality->locality.lb_weight != 0); + // Make sure prorities is big enough. Note that they might not + // arrive in priority order. + if (eds_resource.priorities.size() < parsed_locality->priority + 1) { + eds_resource.priorities.resize(parsed_locality->priority + 1); + } + auto& locality_map = + eds_resource.priorities[parsed_locality->priority].localities; + auto it = locality_map.find(parsed_locality->locality.name.get()); + if (it != locality_map.end()) { + errors.AddError(absl::StrCat( + "duplicate locality ", + parsed_locality->locality.name->AsHumanReadableString(), + " found in priority ", parsed_locality->priority)); + } else { + locality_map.emplace(parsed_locality->locality.name.get(), + std::move(parsed_locality->locality)); + } + } } - } - for (const auto& priority : eds_update->priorities) { - if (priority.localities.empty()) { - errors.push_back( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("sparse priority list")); + for (size_t i = 0; i < eds_resource.priorities.size(); ++i) { + const auto& priority = eds_resource.priorities[i]; + if (priority.localities.empty()) { + errors.AddError(absl::StrCat("priority ", i, " empty")); + } else { + // Check that the sum of the locality weights in this priority + // does not exceed the max value for a uint32. + uint64_t total_weight = 0; + for (const auto& p : priority.localities) { + total_weight += p.second.lb_weight; + if (total_weight > std::numeric_limits<uint32_t>::max()) { + errors.AddError( + absl::StrCat("sum of locality weights for priority ", i, + " exceeds uint32 max")); + break; + } + } + } } } - // Get the drop config. - eds_update->drop_config = MakeRefCounted<XdsEndpointResource::DropConfig>(); - const envoy_config_endpoint_v3_ClusterLoadAssignment_Policy* policy = - envoy_config_endpoint_v3_ClusterLoadAssignment_policy( - cluster_load_assignment); + // policy + eds_resource.drop_config = MakeRefCounted<XdsEndpointResource::DropConfig>(); + const auto* policy = envoy_config_endpoint_v3_ClusterLoadAssignment_policy( + cluster_load_assignment); if (policy != nullptr) { + ValidationErrors::ScopedField field(&errors, "policy"); size_t drop_size; - const envoy_config_endpoint_v3_ClusterLoadAssignment_Policy_DropOverload* const* - drop_overload = - envoy_config_endpoint_v3_ClusterLoadAssignment_Policy_drop_overloads( - policy, &drop_size); - for (size_t j = 0; j < drop_size; ++j) { - grpc_error_handle error = - DropParseAndAppend(drop_overload[j], eds_update->drop_config.get()); - if (error != GRPC_ERROR_NONE) { - errors.push_back( - grpc_error_add_child(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "drop config validation error"), - error)); - } + const auto* const* drop_overload = + envoy_config_endpoint_v3_ClusterLoadAssignment_Policy_drop_overloads( + policy, &drop_size); + for (size_t i = 0; i < drop_size; ++i) { + ValidationErrors::ScopedField field( + &errors, absl::StrCat(".drop_overloads[", i, "]")); + DropParseAndAppend(drop_overload[i], eds_resource.drop_config.get(), + &errors); } } - return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing EDS resource", &errors); + // Return result. + if (!errors.ok()) { + return errors.status(absl::StatusCode::kInvalidArgument, + "errors parsing EDS resource"); + } + return eds_resource; } } // namespace -absl::StatusOr<XdsResourceType::DecodeResult> XdsEndpointResourceType::Decode( - const XdsEncodingContext& context, absl::string_view serialized_resource, - bool is_v2) const { +XdsResourceType::DecodeResult XdsEndpointResourceType::Decode( + const XdsResourceType::DecodeContext& context, + absl::string_view serialized_resource) const { + DecodeResult result; // Parse serialized proto. auto* resource = envoy_config_endpoint_v3_ClusterLoadAssignment_parse( serialized_resource.data(), serialized_resource.size(), context.arena); if (resource == nullptr) { - return absl::InvalidArgumentError( + result.resource = absl::InvalidArgumentError( "Can't parse ClusterLoadAssignment resource."); + return result; } MaybeLogClusterLoadAssignment(context, resource); // Validate resource. - DecodeResult result; result.name = UpbStringToStdString( envoy_config_endpoint_v3_ClusterLoadAssignment_cluster_name(resource)); - auto endpoint_data = absl::make_unique<ResourceDataSubclass>(); - grpc_error_handle error = - EdsResourceParse(context, resource, is_v2, &endpoint_data->resource); - if (error != GRPC_ERROR_NONE) { - std::string error_str = grpc_error_std_string(error); - GRPC_ERROR_UNREF(error); + auto eds_resource = EdsResourceParse(context, resource); + if (!eds_resource.ok()) { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer)) { gpr_log(GPR_ERROR, "[xds_client %p] invalid ClusterLoadAssignment %s: %s", - context.client, result.name.c_str(), error_str.c_str()); + context.client, result.name->c_str(), + eds_resource.status().ToString().c_str()); } - result.resource = absl::InvalidArgumentError(error_str); + result.resource = eds_resource.status(); } else { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer)) { gpr_log(GPR_INFO, "[xds_client %p] parsed ClusterLoadAssignment %s: %s", - context.client, result.name.c_str(), - endpoint_data->resource.ToString().c_str()); + context.client, result.name->c_str(), + eds_resource->ToString().c_str()); } - result.resource = std::move(endpoint_data); + result.resource = + std::make_unique<XdsEndpointResource>(std::move(*eds_resource)); } - return std::move(result); + return result; } } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_endpoint.h b/grpc/src/core/ext/xds/xds_endpoint.h index d3ce9cb0..4c301048 100644 --- a/grpc/src/core/ext/xds/xds_endpoint.h +++ b/grpc/src/core/ext/xds/xds_endpoint.h @@ -14,27 +14,38 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_ENDPOINT_H -#define GRPC_CORE_EXT_XDS_XDS_ENDPOINT_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_ENDPOINT_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_ENDPOINT_H #include <grpc/support/port_platform.h> +#include <stdint.h> + +#include <algorithm> #include <map> -#include <set> +#include <memory> #include <string> +#include <utility> +#include <vector> -#include "absl/container/inlined_vector.h" +#include "absl/base/thread_annotations.h" +#include "absl/random/random.h" +#include "absl/strings/string_view.h" #include "envoy/config/endpoint/v3/endpoint.upbdefs.h" +#include "upb/reflection/def.h" #include "src/core/ext/xds/xds_client.h" #include "src/core/ext/xds/xds_client_stats.h" +#include "src/core/ext/xds/xds_resource_type.h" #include "src/core/ext/xds/xds_resource_type_impl.h" +#include "src/core/lib/gprpp/ref_counted.h" #include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/sync.h" #include "src/core/lib/resolver/server_address.h" namespace grpc_core { -struct XdsEndpointResource { +struct XdsEndpointResource : public XdsResourceType::ResourceData { struct Priority { struct Locality { RefCountedPtr<XdsLocalityName> name; @@ -54,7 +65,7 @@ struct XdsEndpointResource { bool operator==(const Priority& other) const; std::string ToString() const; }; - using PriorityList = absl::InlinedVector<Priority, 2>; + using PriorityList = std::vector<Priority>; // There are two phases of accessing this class's content: // 1. to initialize in the control plane combiner; @@ -72,7 +83,7 @@ struct XdsEndpointResource { const uint32_t parts_per_million; }; - using DropCategoryList = absl::InlinedVector<DropCategory, 2>; + using DropCategoryList = std::vector<DropCategory>; void AddCategory(std::string name, uint32_t parts_per_million) { drop_category_list_.emplace_back( @@ -82,7 +93,7 @@ struct XdsEndpointResource { // The only method invoked from outside the WorkSerializer (used in // the data plane). - bool ShouldDrop(const std::string** category_name) const; + bool ShouldDrop(const std::string** category_name); const DropCategoryList& drop_category_list() const { return drop_category_list_; @@ -100,6 +111,11 @@ struct XdsEndpointResource { private: DropCategoryList drop_category_list_; bool drop_all_ = false; + + // TODO(roth): Consider using a separate thread-local BitGen for each CPU + // to avoid the need for this mutex. + Mutex mu_; + absl::BitGen bit_gen_ ABSL_GUARDED_BY(&mu_); }; PriorityList priorities; @@ -117,19 +133,15 @@ class XdsEndpointResourceType absl::string_view type_url() const override { return "envoy.config.endpoint.v3.ClusterLoadAssignment"; } - absl::string_view v2_type_url() const override { - return "envoy.api.v2.ClusterLoadAssignment"; - } - absl::StatusOr<DecodeResult> Decode(const XdsEncodingContext& context, - absl::string_view serialized_resource, - bool is_v2) const override; + DecodeResult Decode(const XdsResourceType::DecodeContext& context, + absl::string_view serialized_resource) const override; - void InitUpbSymtab(upb_DefPool* symtab) const override { + void InitUpbSymtab(XdsClient*, upb_DefPool* symtab) const override { envoy_config_endpoint_v3_ClusterLoadAssignment_getmsgdef(symtab); } }; } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_ENDPOINT_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_ENDPOINT_H diff --git a/grpc/src/core/ext/xds/xds_health_status.cc b/grpc/src/core/ext/xds/xds_health_status.cc new file mode 100644 index 00000000..e5c3a924 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_health_status.cc @@ -0,0 +1,80 @@ +// +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include <grpc/support/port_platform.h> + +#include "src/core/ext/xds/xds_health_status.h" + +#include "absl/strings/str_cat.h" +#include "envoy/config/core/v3/health_check.upb.h" + +#include "src/core/lib/gpr/useful.h" + +namespace grpc_core { + +absl::optional<XdsHealthStatus> XdsHealthStatus::FromUpb(uint32_t status) { + switch (status) { + case envoy_config_core_v3_UNKNOWN: + return XdsHealthStatus(kUnknown); + case envoy_config_core_v3_HEALTHY: + return XdsHealthStatus(kHealthy); + case envoy_config_core_v3_DRAINING: + return XdsHealthStatus(kDraining); + default: + return absl::nullopt; + } +} + +absl::optional<XdsHealthStatus> XdsHealthStatus::FromString( + absl::string_view status) { + if (status == "UNKNOWN") return XdsHealthStatus(kUnknown); + if (status == "HEALTHY") return XdsHealthStatus(kHealthy); + if (status == "DRAINING") return XdsHealthStatus(kDraining); + return absl::nullopt; +} + +const char* XdsHealthStatus::ToString() const { + switch (status_) { + case kUnknown: + return "UNKNOWN"; + case kHealthy: + return "HEALTHY"; + case kDraining: + return "DRAINING"; + default: + return "<INVALID>"; + } +} + +bool operator<(const XdsHealthStatus& hs1, const XdsHealthStatus& hs2) { + return hs1.status() < hs2.status(); +} + +const char* XdsEndpointHealthStatusAttribute::kKey = + "xds_endpoint_health_status"; + +int XdsEndpointHealthStatusAttribute::Cmp( + const AttributeInterface* other) const { + const auto* other_attr = + static_cast<const XdsEndpointHealthStatusAttribute*>(other); + return QsortCompare(status_, other_attr->status_); +} + +std::string XdsEndpointHealthStatusAttribute::ToString() const { + return absl::StrCat("{status_=", status_.ToString(), "}"); +} + +} // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_health_status.h b/grpc/src/core/ext/xds/xds_health_status.h new file mode 100644 index 00000000..c9c27315 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_health_status.h @@ -0,0 +1,109 @@ +// +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_HEALTH_STATUS_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_HEALTH_STATUS_H + +#include <grpc/support/port_platform.h> + +#include <stdint.h> + +#include <memory> +#include <string> + +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/span.h" + +#include "src/core/lib/resolver/server_address.h" + +namespace grpc_core { + +class XdsHealthStatus { + public: + enum HealthStatus { kUnknown, kHealthy, kDraining }; + + // Returns an XdsHealthStatus for supported enum values, else nullopt. + static absl::optional<XdsHealthStatus> FromUpb(uint32_t status); + static absl::optional<XdsHealthStatus> FromString(absl::string_view status); + + explicit XdsHealthStatus(HealthStatus status) : status_(status) {} + + HealthStatus status() const { return status_; } + + bool operator==(const XdsHealthStatus& other) const { + return status_ == other.status_; + } + + const char* ToString() const; + + private: + HealthStatus status_; +}; + +class XdsHealthStatusSet { + public: + XdsHealthStatusSet() = default; + + explicit XdsHealthStatusSet(absl::Span<const XdsHealthStatus> statuses) { + for (XdsHealthStatus status : statuses) { + Add(status); + } + } + + bool operator==(const XdsHealthStatusSet& other) const { + return status_mask_ == other.status_mask_; + } + + void Clear() { status_mask_ = 0; } + + void Add(XdsHealthStatus status) { status_mask_ |= (0x1 << status.status()); } + + bool Contains(XdsHealthStatus status) const { + return status_mask_ & (0x1 << status.status()); + } + + private: + int status_mask_ = 0; +}; + +bool operator<(const XdsHealthStatus& hs1, const XdsHealthStatus& hs2); + +class XdsEndpointHealthStatusAttribute + : public ServerAddress::AttributeInterface { + public: + static const char* kKey; + + explicit XdsEndpointHealthStatusAttribute(XdsHealthStatus status) + : status_(status) {} + + XdsHealthStatus status() const { return status_; } + + std::unique_ptr<AttributeInterface> Copy() const override { + return std::make_unique<XdsEndpointHealthStatusAttribute>(status_); + } + + int Cmp(const AttributeInterface* other) const override; + + std::string ToString() const override; + + private: + XdsHealthStatus status_; +}; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_HEALTH_STATUS_H diff --git a/grpc/src/core/ext/xds/xds_http_fault_filter.cc b/grpc/src/core/ext/xds/xds_http_fault_filter.cc index 4c7e826c..1950ca81 100644 --- a/grpc/src/core/ext/xds/xds_http_fault_filter.cc +++ b/grpc/src/core/ext/xds/xds_http_fault_filter.cc @@ -18,36 +18,38 @@ #include "src/core/ext/xds/xds_http_fault_filter.h" +#include <stdint.h> + #include <string> +#include <utility> #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" -#include "absl/strings/str_format.h" #include "absl/strings/string_view.h" +#include "absl/types/variant.h" #include "envoy/extensions/filters/common/fault/v3/fault.upb.h" #include "envoy/extensions/filters/http/fault/v3/fault.upb.h" #include "envoy/extensions/filters/http/fault/v3/fault.upbdefs.h" #include "envoy/type/v3/percent.upb.h" -#include "google/protobuf/any.upb.h" -#include "google/protobuf/duration.upb.h" #include "google/protobuf/wrappers.upb.h" -#include "upb/def.h" -#include <grpc/grpc.h> +#include <grpc/status.h> +#include <grpc/support/json.h> #include "src/core/ext/filters/fault_injection/fault_injection_filter.h" +#include "src/core/ext/filters/fault_injection/fault_injection_service_config_parser.h" +#include "src/core/ext/xds/xds_common_types.h" #include "src/core/ext/xds/xds_http_filters.h" #include "src/core/lib/channel/channel_args.h" -#include "src/core/lib/channel/channel_stack.h" #include "src/core/lib/channel/status_util.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/gprpp/validation_errors.h" #include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_writer.h" #include "src/core/lib/transport/status_conversion.h" namespace grpc_core { -const char* kXdsHttpFaultFilterConfigName = - "envoy.extensions.filters.http.fault.v3.HTTPFault"; - namespace { uint32_t GetDenominator(const envoy_type_v3_FractionalPercent* fraction) { @@ -69,13 +71,36 @@ uint32_t GetDenominator(const envoy_type_v3_FractionalPercent* fraction) { return 100; } -absl::StatusOr<Json> ParseHttpFaultIntoJson( - upb_StringView serialized_http_fault, upb_Arena* arena) { +} // namespace + +absl::string_view XdsHttpFaultFilter::ConfigProtoName() const { + return "envoy.extensions.filters.http.fault.v3.HTTPFault"; +} + +absl::string_view XdsHttpFaultFilter::OverrideConfigProtoName() const { + return ""; +} + +void XdsHttpFaultFilter::PopulateSymtab(upb_DefPool* symtab) const { + envoy_extensions_filters_http_fault_v3_HTTPFault_getmsgdef(symtab); +} + +absl::optional<XdsHttpFilterImpl::FilterConfig> +XdsHttpFaultFilter::GenerateFilterConfig( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const { + absl::string_view* serialized_filter_config = + absl::get_if<absl::string_view>(&extension.value); + if (serialized_filter_config == nullptr) { + errors->AddError("could not parse fault injection filter config"); + return absl::nullopt; + } auto* http_fault = envoy_extensions_filters_http_fault_v3_HTTPFault_parse( - serialized_http_fault.data, serialized_http_fault.size, arena); + serialized_filter_config->data(), serialized_filter_config->size(), + context.arena); if (http_fault == nullptr) { - return absl::InvalidArgumentError( - "could not parse fault injection filter config"); + errors->AddError("could not parse fault injection filter config"); + return absl::nullopt; } // NOTE(lidiz): Here, we are manually translating the upb messages into the // JSON form of the filter config as part of method config, which will be @@ -90,6 +115,7 @@ absl::StatusOr<Json> ParseHttpFaultIntoJson( const auto* fault_abort = envoy_extensions_filters_http_fault_v3_HTTPFault_abort(http_fault); if (fault_abort != nullptr) { + ValidationErrors::ScopedField field(errors, ".abort"); grpc_status_code abort_grpc_status_code = GRPC_STATUS_OK; // Try if gRPC status code is set first int abort_grpc_status_code_raw = @@ -98,68 +124,75 @@ absl::StatusOr<Json> ParseHttpFaultIntoJson( if (abort_grpc_status_code_raw != 0) { if (!grpc_status_code_from_int(abort_grpc_status_code_raw, &abort_grpc_status_code)) { - return absl::InvalidArgumentError(absl::StrCat( - "invalid gRPC status code: ", abort_grpc_status_code_raw)); + ValidationErrors::ScopedField field(errors, ".grpc_status"); + errors->AddError(absl::StrCat("invalid gRPC status code: ", + abort_grpc_status_code_raw)); } } else { // if gRPC status code is empty, check http status int abort_http_status_code = envoy_extensions_filters_http_fault_v3_FaultAbort_http_status( fault_abort); - if (abort_http_status_code != 0 and abort_http_status_code != 200) { + if (abort_http_status_code != 0 && abort_http_status_code != 200) { abort_grpc_status_code = grpc_http2_status_to_grpc_status(abort_http_status_code); } } // Set the abort_code, even if it's OK fault_injection_policy_json["abortCode"] = - grpc_status_code_to_string(abort_grpc_status_code); + Json::FromString(grpc_status_code_to_string(abort_grpc_status_code)); // Set the headers if we enabled header abort injection control if (envoy_extensions_filters_http_fault_v3_FaultAbort_has_header_abort( fault_abort)) { fault_injection_policy_json["abortCodeHeader"] = - "x-envoy-fault-abort-grpc-request"; + Json::FromString("x-envoy-fault-abort-grpc-request"); fault_injection_policy_json["abortPercentageHeader"] = - "x-envoy-fault-abort-percentage"; + Json::FromString("x-envoy-fault-abort-percentage"); } // Set the fraction percent auto* percent = envoy_extensions_filters_http_fault_v3_FaultAbort_percentage( fault_abort); - fault_injection_policy_json["abortPercentageNumerator"] = - Json(envoy_type_v3_FractionalPercent_numerator(percent)); - fault_injection_policy_json["abortPercentageDenominator"] = - Json(GetDenominator(percent)); + if (percent != nullptr) { + fault_injection_policy_json["abortPercentageNumerator"] = + Json::FromNumber(envoy_type_v3_FractionalPercent_numerator(percent)); + fault_injection_policy_json["abortPercentageDenominator"] = + Json::FromNumber(GetDenominator(percent)); + } } // Section 2: Parse the delay injection config const auto* fault_delay = envoy_extensions_filters_http_fault_v3_HTTPFault_delay(http_fault); if (fault_delay != nullptr) { + ValidationErrors::ScopedField field(errors, ".delay"); // Parse the delay duration const auto* delay_duration = envoy_extensions_filters_common_fault_v3_FaultDelay_fixed_delay( fault_delay); if (delay_duration != nullptr) { - fault_injection_policy_json["delay"] = absl::StrFormat( - "%d.%09ds", google_protobuf_Duration_seconds(delay_duration), - google_protobuf_Duration_nanos(delay_duration)); + ValidationErrors::ScopedField field(errors, ".fixed_delay"); + Duration duration = ParseDuration(delay_duration, errors); + fault_injection_policy_json["delay"] = + Json::FromString(duration.ToJsonString()); } // Set the headers if we enabled header delay injection control if (envoy_extensions_filters_common_fault_v3_FaultDelay_has_header_delay( fault_delay)) { fault_injection_policy_json["delayHeader"] = - "x-envoy-fault-delay-request"; + Json::FromString("x-envoy-fault-delay-request"); fault_injection_policy_json["delayPercentageHeader"] = - "x-envoy-fault-delay-request-percentage"; + Json::FromString("x-envoy-fault-delay-request-percentage"); } // Set the fraction percent auto* percent = envoy_extensions_filters_common_fault_v3_FaultDelay_percentage( fault_delay); - fault_injection_policy_json["delayPercentageNumerator"] = - Json(envoy_type_v3_FractionalPercent_numerator(percent)); - fault_injection_policy_json["delayPercentageDenominator"] = - Json(GetDenominator(percent)); + if (percent != nullptr) { + fault_injection_policy_json["delayPercentageNumerator"] = + Json::FromNumber(envoy_type_v3_FractionalPercent_numerator(percent)); + fault_injection_policy_json["delayPercentageDenominator"] = + Json::FromNumber(GetDenominator(percent)); + } } // Section 3: Parse the maximum active faults const auto* max_fault_wrapper = @@ -167,61 +200,40 @@ absl::StatusOr<Json> ParseHttpFaultIntoJson( http_fault); if (max_fault_wrapper != nullptr) { fault_injection_policy_json["maxFaults"] = - google_protobuf_UInt32Value_value(max_fault_wrapper); - } - return fault_injection_policy_json; -} - -} // namespace - -void XdsHttpFaultFilter::PopulateSymtab(upb_DefPool* symtab) const { - envoy_extensions_filters_http_fault_v3_HTTPFault_getmsgdef(symtab); -} - -absl::StatusOr<XdsHttpFilterImpl::FilterConfig> -XdsHttpFaultFilter::GenerateFilterConfig( - upb_StringView serialized_filter_config, upb_Arena* arena) const { - absl::StatusOr<Json> parse_result = - ParseHttpFaultIntoJson(serialized_filter_config, arena); - if (!parse_result.ok()) { - return parse_result.status(); + Json::FromNumber(google_protobuf_UInt32Value_value(max_fault_wrapper)); } - return FilterConfig{kXdsHttpFaultFilterConfigName, std::move(*parse_result)}; + return FilterConfig{ConfigProtoName(), + Json::FromObject(std::move(fault_injection_policy_json))}; } -absl::StatusOr<XdsHttpFilterImpl::FilterConfig> +absl::optional<XdsHttpFilterImpl::FilterConfig> XdsHttpFaultFilter::GenerateFilterConfigOverride( - upb_StringView serialized_filter_config, upb_Arena* arena) const { + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const { // HTTPFault filter has the same message type in HTTP connection manager's // filter config and in overriding filter config field. - return GenerateFilterConfig(serialized_filter_config, arena); + return GenerateFilterConfig(context, std::move(extension), errors); } const grpc_channel_filter* XdsHttpFaultFilter::channel_filter() const { return &FaultInjectionFilter::kFilter; } -grpc_channel_args* XdsHttpFaultFilter::ModifyChannelArgs( - grpc_channel_args* args) const { - grpc_arg args_to_add = grpc_channel_arg_integer_create( - const_cast<char*>(GRPC_ARG_PARSE_FAULT_INJECTION_METHOD_CONFIG), 1); - grpc_channel_args* new_args = - grpc_channel_args_copy_and_add(args, &args_to_add, 1); - // Since this function takes the ownership of the channel args, it needs to - // deallocate the old ones to prevent leak. - grpc_channel_args_destroy(args); - return new_args; +ChannelArgs XdsHttpFaultFilter::ModifyChannelArgs( + const ChannelArgs& args) const { + return args.Set(GRPC_ARG_PARSE_FAULT_INJECTION_METHOD_CONFIG, 1); } absl::StatusOr<XdsHttpFilterImpl::ServiceConfigJsonEntry> XdsHttpFaultFilter::GenerateServiceConfig( const FilterConfig& hcm_filter_config, - const FilterConfig* filter_config_override) const { + const FilterConfig* filter_config_override, + absl::string_view /*filter_name*/) const { Json policy_json = filter_config_override != nullptr ? filter_config_override->config : hcm_filter_config.config; // The policy JSON may be empty, that's allowed. - return ServiceConfigJsonEntry{"faultInjectionPolicy", policy_json.Dump()}; + return ServiceConfigJsonEntry{"faultInjectionPolicy", JsonDump(policy_json)}; } } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_http_fault_filter.h b/grpc/src/core/ext/xds/xds_http_fault_filter.h index edc46e12..f2dd3555 100644 --- a/grpc/src/core/ext/xds/xds_http_fault_filter.h +++ b/grpc/src/core/ext/xds/xds_http_fault_filter.h @@ -14,51 +14,46 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_HTTP_FAULT_FILTER_H -#define GRPC_CORE_EXT_XDS_XDS_HTTP_FAULT_FILTER_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_FAULT_FILTER_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_FAULT_FILTER_H #include <grpc/support/port_platform.h> #include "absl/status/statusor.h" -#include "upb/def.h" - -#include <grpc/grpc.h> +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "upb/reflection/def.h" +#include "src/core/ext/xds/xds_common_types.h" #include "src/core/ext/xds/xds_http_filters.h" +#include "src/core/ext/xds/xds_resource_type.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_fwd.h" +#include "src/core/lib/gprpp/validation_errors.h" namespace grpc_core { -extern const char* kXdsHttpFaultFilterConfigName; - class XdsHttpFaultFilter : public XdsHttpFilterImpl { public: - // Overrides the PopulateSymtab method + absl::string_view ConfigProtoName() const override; + absl::string_view OverrideConfigProtoName() const override; void PopulateSymtab(upb_DefPool* symtab) const override; - - // Overrides the GenerateFilterConfig method - absl::StatusOr<FilterConfig> GenerateFilterConfig( - upb_StringView serialized_filter_config, upb_Arena* arena) const override; - - // Overrides the GenerateFilterConfigOverride method - absl::StatusOr<FilterConfig> GenerateFilterConfigOverride( - upb_StringView serialized_filter_config, upb_Arena* arena) const override; - - // Overrides the channel_filter method + absl::optional<FilterConfig> GenerateFilterConfig( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const override; + absl::optional<FilterConfig> GenerateFilterConfigOverride( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const override; const grpc_channel_filter* channel_filter() const override; - - // Overrides the ModifyChannelArgs method - grpc_channel_args* ModifyChannelArgs(grpc_channel_args* args) const override; - - // Overrides the GenerateServiceConfig method + ChannelArgs ModifyChannelArgs(const ChannelArgs& args) const override; absl::StatusOr<ServiceConfigJsonEntry> GenerateServiceConfig( const FilterConfig& hcm_filter_config, - const FilterConfig* filter_config_override) const override; - + const FilterConfig* filter_config_override, + absl::string_view filter_name) const override; bool IsSupportedOnClients() const override { return true; } - bool IsSupportedOnServers() const override { return false; } }; } // namespace grpc_core -#endif /* GRPC_CORE_EXT_XDS_XDS_HTTP_FAULT_FILTER_H */ +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_FAULT_FILTER_H diff --git a/grpc/src/core/ext/xds/xds_http_filters.cc b/grpc/src/core/ext/xds/xds_http_filters.cc index f2f28144..96ced9d7 100644 --- a/grpc/src/core/ext/xds/xds_http_filters.cc +++ b/grpc/src/core/ext/xds/xds_http_filters.cc @@ -18,105 +18,104 @@ #include "src/core/ext/xds/xds_http_filters.h" +#include <algorithm> +#include <map> +#include <utility> +#include <vector> + +#include "absl/types/variant.h" #include "envoy/extensions/filters/http/router/v3/router.upb.h" #include "envoy/extensions/filters/http/router/v3/router.upbdefs.h" +#include <grpc/support/log.h> + +#include "src/core/ext/xds/xds_cluster.h" #include "src/core/ext/xds/xds_http_fault_filter.h" #include "src/core/ext/xds/xds_http_rbac_filter.h" +#include "src/core/ext/xds/xds_http_stateful_session_filter.h" namespace grpc_core { -const char* kXdsHttpRouterFilterConfigName = - "envoy.extensions.filters.http.router.v3.Router"; +// +// XdsHttpRouterFilter +// -namespace { +absl::string_view XdsHttpRouterFilter::ConfigProtoName() const { + return "envoy.extensions.filters.http.router.v3.Router"; +} -class XdsHttpRouterFilter : public XdsHttpFilterImpl { - public: - void PopulateSymtab(upb_DefPool* symtab) const override { - envoy_extensions_filters_http_router_v3_Router_getmsgdef(symtab); - } +absl::string_view XdsHttpRouterFilter::OverrideConfigProtoName() const { + return ""; +} - absl::StatusOr<FilterConfig> GenerateFilterConfig( - upb_StringView serialized_filter_config, - upb_Arena* arena) const override { - if (envoy_extensions_filters_http_router_v3_Router_parse( - serialized_filter_config.data, serialized_filter_config.size, - arena) == nullptr) { - return absl::InvalidArgumentError("could not parse router filter config"); - } - return FilterConfig{kXdsHttpRouterFilterConfigName, Json()}; - } +void XdsHttpRouterFilter::PopulateSymtab(upb_DefPool* symtab) const { + envoy_extensions_filters_http_router_v3_Router_getmsgdef(symtab); +} - absl::StatusOr<FilterConfig> GenerateFilterConfigOverride( - upb_StringView /*serialized_filter_config*/, - upb_Arena* /*arena*/) const override { - return absl::InvalidArgumentError( - "router filter does not support config override"); +absl::optional<XdsHttpFilterImpl::FilterConfig> +XdsHttpRouterFilter::GenerateFilterConfig( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const { + absl::string_view* serialized_filter_config = + absl::get_if<absl::string_view>(&extension.value); + if (serialized_filter_config == nullptr) { + errors->AddError("could not parse router filter config"); + return absl::nullopt; } - - const grpc_channel_filter* channel_filter() const override { return nullptr; } - - // No-op. This will never be called, since channel_filter() returns null. - absl::StatusOr<ServiceConfigJsonEntry> GenerateServiceConfig( - const FilterConfig& /*hcm_filter_config*/, - const FilterConfig* /*filter_config_override*/) const override { - return absl::UnimplementedError("router filter should never be called"); + if (envoy_extensions_filters_http_router_v3_Router_parse( + serialized_filter_config->data(), serialized_filter_config->size(), + context.arena) == nullptr) { + errors->AddError("could not parse router filter config"); + return absl::nullopt; } + return FilterConfig{ConfigProtoName(), Json()}; +} - bool IsSupportedOnClients() const override { return true; } - - bool IsSupportedOnServers() const override { return true; } - - bool IsTerminalFilter() const override { return true; } -}; - -using FilterOwnerList = std::vector<std::unique_ptr<XdsHttpFilterImpl>>; -using FilterRegistryMap = std::map<absl::string_view, XdsHttpFilterImpl*>; +absl::optional<XdsHttpFilterImpl::FilterConfig> +XdsHttpRouterFilter::GenerateFilterConfigOverride( + const XdsResourceType::DecodeContext& /*context*/, + XdsExtension /*extension*/, ValidationErrors* errors) const { + errors->AddError("router filter does not support config override"); + return absl::nullopt; +} -FilterOwnerList* g_filters = nullptr; -FilterRegistryMap* g_filter_registry = nullptr; +// +// XdsHttpFilterRegistry +// -} // namespace +XdsHttpFilterRegistry::XdsHttpFilterRegistry(bool register_builtins) { + if (register_builtins) { + RegisterFilter(std::make_unique<XdsHttpRouterFilter>()); + RegisterFilter(std::make_unique<XdsHttpFaultFilter>()); + RegisterFilter(std::make_unique<XdsHttpRbacFilter>()); + if (XdsOverrideHostEnabled()) { + RegisterFilter(std::make_unique<XdsHttpStatefulSessionFilter>()); + } + } +} void XdsHttpFilterRegistry::RegisterFilter( - std::unique_ptr<XdsHttpFilterImpl> filter, - const std::set<absl::string_view>& config_proto_type_names) { - for (auto config_proto_type_name : config_proto_type_names) { - (*g_filter_registry)[config_proto_type_name] = filter.get(); + std::unique_ptr<XdsHttpFilterImpl> filter) { + GPR_ASSERT( + registry_map_.emplace(filter->ConfigProtoName(), filter.get()).second); + auto override_proto_name = filter->OverrideConfigProtoName(); + if (!override_proto_name.empty()) { + GPR_ASSERT(registry_map_.emplace(override_proto_name, filter.get()).second); } - g_filters->push_back(std::move(filter)); + owning_list_.push_back(std::move(filter)); } const XdsHttpFilterImpl* XdsHttpFilterRegistry::GetFilterForType( - absl::string_view proto_type_name) { - auto it = g_filter_registry->find(proto_type_name); - if (it == g_filter_registry->end()) return nullptr; + absl::string_view proto_type_name) const { + auto it = registry_map_.find(proto_type_name); + if (it == registry_map_.end()) return nullptr; return it->second; } -void XdsHttpFilterRegistry::PopulateSymtab(upb_DefPool* symtab) { - for (const auto& filter : *g_filters) { +void XdsHttpFilterRegistry::PopulateSymtab(upb_DefPool* symtab) const { + for (const auto& filter : owning_list_) { filter->PopulateSymtab(symtab); } } -void XdsHttpFilterRegistry::Init() { - g_filters = new FilterOwnerList; - g_filter_registry = new FilterRegistryMap; - RegisterFilter(absl::make_unique<XdsHttpRouterFilter>(), - {kXdsHttpRouterFilterConfigName}); - RegisterFilter(absl::make_unique<XdsHttpFaultFilter>(), - {kXdsHttpFaultFilterConfigName}); - RegisterFilter(absl::make_unique<XdsHttpRbacFilter>(), - {kXdsHttpRbacFilterConfigName}); - RegisterFilter(absl::make_unique<XdsHttpRbacFilter>(), - {kXdsHttpRbacFilterConfigOverrideName}); -} - -void XdsHttpFilterRegistry::Shutdown() { - delete g_filter_registry; - delete g_filters; -} - } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_http_filters.h b/grpc/src/core/ext/xds/xds_http_filters.h index 5500bff2..26d73cd7 100644 --- a/grpc/src/core/ext/xds/xds_http_filters.h +++ b/grpc/src/core/ext/xds/xds_http_filters.h @@ -14,30 +14,34 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_HTTP_FILTERS_H -#define GRPC_CORE_EXT_XDS_XDS_HTTP_FILTERS_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_FILTERS_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_FILTERS_H #include <grpc/support/port_platform.h> +#include <map> #include <memory> -#include <set> #include <string> +#include <utility> +#include <vector> +#include "absl/status/status.h" #include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/string_view.h" -#include "google/protobuf/any.upb.h" -#include "upb/def.h" - -#include <grpc/grpc.h> - -#include "src/core/lib/channel/channel_stack.h" +#include "absl/types/optional.h" +#include "upb/reflection/def.h" + +#include "src/core/ext/xds/xds_common_types.h" +#include "src/core/ext/xds/xds_resource_type.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_fwd.h" +#include "src/core/lib/gprpp/validation_errors.h" #include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_writer.h" namespace grpc_core { -extern const char* kXdsHttpRouterFilterConfigName; - class XdsHttpFilterImpl { public: struct FilterConfig { @@ -50,7 +54,7 @@ class XdsHttpFilterImpl { } std::string ToString() const { return absl::StrCat("{config_proto_type_name=", config_proto_type_name, - " config=", config.Dump(), "}"); + " config=", JsonDump(config), "}"); } }; @@ -69,26 +73,34 @@ class XdsHttpFilterImpl { virtual ~XdsHttpFilterImpl() = default; + // Returns the top-level filter config proto message name. + virtual absl::string_view ConfigProtoName() const = 0; + + // Returns the override filter config proto message name. + // If empty, no override type is supported. + virtual absl::string_view OverrideConfigProtoName() const = 0; + // Loads the proto message into the upb symtab. virtual void PopulateSymtab(upb_DefPool* symtab) const = 0; // Generates a Config from the xDS filter config proto. // Used for the top-level config in the HCM HTTP filter list. - virtual absl::StatusOr<FilterConfig> GenerateFilterConfig( - upb_StringView serialized_filter_config, upb_Arena* arena) const = 0; + virtual absl::optional<FilterConfig> GenerateFilterConfig( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const = 0; // Generates a Config from the xDS filter config proto. // Used for the typed_per_filter_config override in VirtualHost and Route. - virtual absl::StatusOr<FilterConfig> GenerateFilterConfigOverride( - upb_StringView serialized_filter_config, upb_Arena* arena) const = 0; + virtual absl::optional<FilterConfig> GenerateFilterConfigOverride( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const = 0; // C-core channel filter implementation. virtual const grpc_channel_filter* channel_filter() const = 0; // Modifies channel args that may affect service config parsing (not // visible to the channel as a whole). - // Takes ownership of args. Caller takes ownership of return value. - virtual grpc_channel_args* ModifyChannelArgs(grpc_channel_args* args) const { + virtual ChannelArgs ModifyChannelArgs(const ChannelArgs& args) const { return args; } @@ -100,7 +112,8 @@ class XdsHttpFilterImpl { // there is no override in any of those locations. virtual absl::StatusOr<ServiceConfigJsonEntry> GenerateServiceConfig( const FilterConfig& hcm_filter_config, - const FilterConfig* filter_config_override) const = 0; + const FilterConfig* filter_config_override, + absl::string_view filter_name) const = 0; // Returns true if the filter is supported on clients; false otherwise virtual bool IsSupportedOnClients() const = 0; @@ -112,22 +125,60 @@ class XdsHttpFilterImpl { virtual bool IsTerminalFilter() const { return false; } }; +class XdsHttpRouterFilter : public XdsHttpFilterImpl { + public: + absl::string_view ConfigProtoName() const override; + absl::string_view OverrideConfigProtoName() const override; + void PopulateSymtab(upb_DefPool* symtab) const override; + absl::optional<FilterConfig> GenerateFilterConfig( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const override; + absl::optional<FilterConfig> GenerateFilterConfigOverride( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const override; + const grpc_channel_filter* channel_filter() const override { return nullptr; } + absl::StatusOr<ServiceConfigJsonEntry> GenerateServiceConfig( + const FilterConfig& /*hcm_filter_config*/, + const FilterConfig* /*filter_config_override*/, + absl::string_view /*filter_name*/) const override { + // This will never be called, since channel_filter() returns null. + return absl::UnimplementedError("router filter should never be called"); + } + bool IsSupportedOnClients() const override { return true; } + bool IsSupportedOnServers() const override { return true; } + bool IsTerminalFilter() const override { return true; } +}; + class XdsHttpFilterRegistry { public: - static void RegisterFilter( - std::unique_ptr<XdsHttpFilterImpl> filter, - const std::set<absl::string_view>& config_proto_type_names); + explicit XdsHttpFilterRegistry(bool register_builtins = true); + + // Not copyable. + XdsHttpFilterRegistry(const XdsHttpFilterRegistry&) = delete; + XdsHttpFilterRegistry& operator=(const XdsHttpFilterRegistry&) = delete; + + // Movable. + XdsHttpFilterRegistry(XdsHttpFilterRegistry&& other) noexcept + : owning_list_(std::move(other.owning_list_)), + registry_map_(std::move(other.registry_map_)) {} + XdsHttpFilterRegistry& operator=(XdsHttpFilterRegistry&& other) noexcept { + owning_list_ = std::move(other.owning_list_); + registry_map_ = std::move(other.registry_map_); + return *this; + } + + void RegisterFilter(std::unique_ptr<XdsHttpFilterImpl> filter); - static const XdsHttpFilterImpl* GetFilterForType( - absl::string_view proto_type_name); + const XdsHttpFilterImpl* GetFilterForType( + absl::string_view proto_type_name) const; - static void PopulateSymtab(upb_DefPool* symtab); + void PopulateSymtab(upb_DefPool* symtab) const; - // Global init and shutdown. - static void Init(); - static void Shutdown(); + private: + std::vector<std::unique_ptr<XdsHttpFilterImpl>> owning_list_; + std::map<absl::string_view, XdsHttpFilterImpl*> registry_map_; }; } // namespace grpc_core -#endif /* GRPC_CORE_EXT_XDS_XDS_HTTP_FILTERS_H */ +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_FILTERS_H diff --git a/grpc/src/core/ext/xds/xds_http_rbac_filter.cc b/grpc/src/core/ext/xds/xds_http_rbac_filter.cc index 14cbbd30..f7bde62b 100644 --- a/grpc/src/core/ext/xds/xds_http_rbac_filter.cc +++ b/grpc/src/core/ext/xds/xds_http_rbac_filter.cc @@ -18,7 +18,17 @@ #include "src/core/ext/xds/xds_http_rbac_filter.h" -#include "absl/strings/str_format.h" +#include <stddef.h> +#include <stdint.h> + +#include <algorithm> +#include <string> +#include <utility> + +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" +#include "absl/types/variant.h" #include "envoy/config/core/v3/address.upb.h" #include "envoy/config/rbac/v3/rbac.upb.h" #include "envoy/config/route/v3/route_components.upb.h" @@ -30,53 +40,68 @@ #include "envoy/type/matcher/v3/string.upb.h" #include "envoy/type/v3/range.upb.h" #include "google/protobuf/wrappers.upb.h" +#include "upb/collections/map.h" + +#include <grpc/support/json.h> #include "src/core/ext/filters/rbac/rbac_filter.h" #include "src/core/ext/filters/rbac/rbac_service_config_parser.h" #include "src/core/ext/xds/upb_utils.h" +#include "src/core/ext/xds/xds_audit_logger_registry.h" +#include "src/core/ext/xds/xds_bootstrap_grpc.h" +#include "src/core/ext/xds/xds_client.h" #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gpr/string.h" +#include "src/core/lib/gprpp/env.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_writer.h" namespace grpc_core { -const char* kXdsHttpRbacFilterConfigName = - "envoy.extensions.filters.http.rbac.v3.RBAC"; - -const char* kXdsHttpRbacFilterConfigOverrideName = - "envoy.extensions.filters.http.rbac.v3.RBACPerRoute"; - namespace { +// TODO(lwge): Remove once the feature is stable. +bool XdsRbacAuditLoggingEnabled() { + auto value = GetEnv("GRPC_EXPERIMENTAL_XDS_RBAC_AUDIT_LOGGING"); + if (!value.has_value()) return false; + bool parsed_value; + bool parse_succeeded = gpr_parse_bool_value(value->c_str(), &parsed_value); + return parse_succeeded && parsed_value; +} + Json ParseRegexMatcherToJson( const envoy_type_matcher_v3_RegexMatcher* regex_matcher) { - return Json::Object( - {{"regex", UpbStringToStdString(envoy_type_matcher_v3_RegexMatcher_regex( - regex_matcher))}}); + return Json::FromObject( + {{"regex", + Json::FromString(UpbStringToStdString( + envoy_type_matcher_v3_RegexMatcher_regex(regex_matcher)))}}); } Json ParseInt64RangeToJson(const envoy_type_v3_Int64Range* range) { - return Json::Object{{"start", envoy_type_v3_Int64Range_start(range)}, - {"end", envoy_type_v3_Int64Range_end(range)}}; + return Json::FromObject( + {{"start", Json::FromNumber(envoy_type_v3_Int64Range_start(range))}, + {"end", Json::FromNumber(envoy_type_v3_Int64Range_end(range))}}); } -absl::StatusOr<Json> ParseHeaderMatcherToJson( - const envoy_config_route_v3_HeaderMatcher* header) { +Json ParseHeaderMatcherToJson(const envoy_config_route_v3_HeaderMatcher* header, + ValidationErrors* errors) { Json::Object header_json; - std::vector<absl::Status> error_list; - std::string name = - UpbStringToStdString(envoy_config_route_v3_HeaderMatcher_name(header)); - if (name == ":scheme") { - error_list.push_back( - absl::InvalidArgumentError("':scheme' not allowed in header")); - } else if (absl::StartsWith(name, "grpc-")) { - error_list.push_back( - absl::InvalidArgumentError("'grpc-' prefixes not allowed in header")); + { + ValidationErrors::ScopedField field(errors, ".name"); + std::string name = + UpbStringToStdString(envoy_config_route_v3_HeaderMatcher_name(header)); + if (name == ":scheme") { + errors->AddError("':scheme' not allowed in header"); + } else if (absl::StartsWith(name, "grpc-")) { + errors->AddError("'grpc-' prefixes not allowed in header"); + } + header_json.emplace("name", Json::FromString(std::move(name))); } - header_json.emplace("name", std::move(name)); if (envoy_config_route_v3_HeaderMatcher_has_exact_match(header)) { header_json.emplace( "exactMatch", - UpbStringToStdString( - envoy_config_route_v3_HeaderMatcher_exact_match(header))); + Json::FromString(UpbStringToStdString( + envoy_config_route_v3_HeaderMatcher_exact_match(header)))); } else if (envoy_config_route_v3_HeaderMatcher_has_safe_regex_match(header)) { header_json.emplace( "safeRegexMatch", @@ -90,169 +115,146 @@ absl::StatusOr<Json> ParseHeaderMatcherToJson( } else if (envoy_config_route_v3_HeaderMatcher_has_present_match(header)) { header_json.emplace( "presentMatch", - envoy_config_route_v3_HeaderMatcher_present_match(header)); + Json::FromBool( + envoy_config_route_v3_HeaderMatcher_present_match(header))); } else if (envoy_config_route_v3_HeaderMatcher_has_prefix_match(header)) { header_json.emplace( "prefixMatch", - UpbStringToStdString( - envoy_config_route_v3_HeaderMatcher_prefix_match(header))); + Json::FromString(UpbStringToStdString( + envoy_config_route_v3_HeaderMatcher_prefix_match(header)))); } else if (envoy_config_route_v3_HeaderMatcher_has_suffix_match(header)) { header_json.emplace( "suffixMatch", - UpbStringToStdString( - envoy_config_route_v3_HeaderMatcher_suffix_match(header))); + Json::FromString(UpbStringToStdString( + envoy_config_route_v3_HeaderMatcher_suffix_match(header)))); } else if (envoy_config_route_v3_HeaderMatcher_has_contains_match(header)) { header_json.emplace( "containsMatch", - UpbStringToStdString( - envoy_config_route_v3_HeaderMatcher_contains_match(header))); + Json::FromString(UpbStringToStdString( + envoy_config_route_v3_HeaderMatcher_contains_match(header)))); } else { - error_list.push_back( - absl::InvalidArgumentError("Invalid route header matcher specified.")); - } - if (!error_list.empty()) { - return StatusCreate(absl::StatusCode::kInvalidArgument, - "Error parsing HeaderMatcher", DEBUG_LOCATION, - std::move(error_list)); + errors->AddError("invalid route header matcher specified"); } - header_json.emplace("invertMatch", - envoy_config_route_v3_HeaderMatcher_invert_match(header)); - return header_json; + header_json.emplace( + "invertMatch", + Json::FromBool(envoy_config_route_v3_HeaderMatcher_invert_match(header))); + return Json::FromObject(std::move(header_json)); } -absl::StatusOr<Json> ParseStringMatcherToJson( - const envoy_type_matcher_v3_StringMatcher* matcher) { +Json ParseStringMatcherToJson( + const envoy_type_matcher_v3_StringMatcher* matcher, + ValidationErrors* errors) { Json::Object json; if (envoy_type_matcher_v3_StringMatcher_has_exact(matcher)) { json.emplace("exact", - UpbStringToStdString( - envoy_type_matcher_v3_StringMatcher_exact(matcher))); + Json::FromString(UpbStringToStdString( + envoy_type_matcher_v3_StringMatcher_exact(matcher)))); } else if (envoy_type_matcher_v3_StringMatcher_has_prefix(matcher)) { json.emplace("prefix", - UpbStringToStdString( - envoy_type_matcher_v3_StringMatcher_prefix(matcher))); + Json::FromString(UpbStringToStdString( + envoy_type_matcher_v3_StringMatcher_prefix(matcher)))); } else if (envoy_type_matcher_v3_StringMatcher_has_suffix(matcher)) { json.emplace("suffix", - UpbStringToStdString( - envoy_type_matcher_v3_StringMatcher_suffix(matcher))); + Json::FromString(UpbStringToStdString( + envoy_type_matcher_v3_StringMatcher_suffix(matcher)))); } else if (envoy_type_matcher_v3_StringMatcher_has_safe_regex(matcher)) { json.emplace("safeRegex", ParseRegexMatcherToJson( envoy_type_matcher_v3_StringMatcher_safe_regex(matcher))); } else if (envoy_type_matcher_v3_StringMatcher_has_contains(matcher)) { json.emplace("contains", - UpbStringToStdString( - envoy_type_matcher_v3_StringMatcher_contains(matcher))); + Json::FromString(UpbStringToStdString( + envoy_type_matcher_v3_StringMatcher_contains(matcher)))); } else { - return absl::InvalidArgumentError("StringMatcher: Invalid match pattern"); + errors->AddError("invalid match pattern"); } - json.emplace("ignoreCase", - envoy_type_matcher_v3_StringMatcher_ignore_case(matcher)); - return json; + json.emplace( + "ignoreCase", + Json::FromBool(envoy_type_matcher_v3_StringMatcher_ignore_case(matcher))); + return Json::FromObject(std::move(json)); } -absl::StatusOr<Json> ParsePathMatcherToJson( - const envoy_type_matcher_v3_PathMatcher* matcher) { +Json ParsePathMatcherToJson(const envoy_type_matcher_v3_PathMatcher* matcher, + ValidationErrors* errors) { + ValidationErrors::ScopedField field(errors, ".path"); const auto* path = envoy_type_matcher_v3_PathMatcher_path(matcher); if (path == nullptr) { - return absl::InvalidArgumentError("PathMatcher has empty path"); - } - Json::Object json; - auto path_json = ParseStringMatcherToJson(path); - if (!path_json.ok()) { - return path_json; + errors->AddError("field not present"); + return Json(); } - json.emplace("path", std::move(*path_json)); - return json; -} - -Json ParseUInt32ValueToJson(const google_protobuf_UInt32Value* value) { - return Json::Object{{"value", google_protobuf_UInt32Value_value(value)}}; + Json path_json = ParseStringMatcherToJson(path, errors); + return Json::FromObject({{"path", std::move(path_json)}}); } Json ParseCidrRangeToJson(const envoy_config_core_v3_CidrRange* range) { Json::Object json; json.emplace("addressPrefix", - UpbStringToStdString( - envoy_config_core_v3_CidrRange_address_prefix(range))); + Json::FromString(UpbStringToStdString( + envoy_config_core_v3_CidrRange_address_prefix(range)))); const auto* prefix_len = envoy_config_core_v3_CidrRange_prefix_len(range); if (prefix_len != nullptr) { - json.emplace("prefixLen", ParseUInt32ValueToJson(prefix_len)); + json.emplace( + "prefixLen", + Json::FromNumber(google_protobuf_UInt32Value_value(prefix_len))); } - return json; + return Json::FromObject(std::move(json)); } Json ParseMetadataMatcherToJson( const envoy_type_matcher_v3_MetadataMatcher* metadata_matcher) { - Json::Object json; // The fields "filter", "path" and "value" are irrelevant to gRPC as per // https://github.com/grpc/proposal/blob/master/A41-xds-rbac.md and are not // being parsed. - json.emplace("invert", - envoy_type_matcher_v3_MetadataMatcher_invert(metadata_matcher)); - return json; + return Json::FromObject({ + {"invert", Json::FromBool(envoy_type_matcher_v3_MetadataMatcher_invert( + metadata_matcher))}, + }); } -absl::StatusOr<Json> ParsePermissionToJson( - const envoy_config_rbac_v3_Permission* permission) { +Json ParsePermissionToJson(const envoy_config_rbac_v3_Permission* permission, + ValidationErrors* errors) { Json::Object permission_json; // Helper function to parse Permission::Set to JSON. Used by `and_rules` and // `or_rules`. auto parse_permission_set_to_json = - [](const envoy_config_rbac_v3_Permission_Set* set) - -> absl::StatusOr<Json> { - std::vector<absl::Status> error_list; + [errors](const envoy_config_rbac_v3_Permission_Set* set) -> Json { Json::Array rules_json; size_t size; const envoy_config_rbac_v3_Permission* const* rules = envoy_config_rbac_v3_Permission_Set_rules(set, &size); for (size_t i = 0; i < size; ++i) { - auto permission_json = ParsePermissionToJson(rules[i]); - if (!permission_json.ok()) { - error_list.push_back(permission_json.status()); - } else { - rules_json.emplace_back(std::move(*permission_json)); - } + ValidationErrors::ScopedField field(errors, + absl::StrCat(".rules[", i, "]")); + Json permission_json = ParsePermissionToJson(rules[i], errors); + rules_json.emplace_back(std::move(permission_json)); } - if (!error_list.empty()) { - return StatusCreate(absl::StatusCode::kInvalidArgument, - "Error parsing Set", DEBUG_LOCATION, - std::move(error_list)); - } - return Json::Object({{"rules", std::move(rules_json)}}); + return Json::FromObject( + {{"rules", Json::FromArray(std::move(rules_json))}}); }; if (envoy_config_rbac_v3_Permission_has_and_rules(permission)) { + ValidationErrors::ScopedField field(errors, ".and_permission"); const auto* and_rules = envoy_config_rbac_v3_Permission_and_rules(permission); - auto permission_set_json = parse_permission_set_to_json(and_rules); - if (!permission_set_json.ok()) { - return permission_set_json; - } - permission_json.emplace("andRules", std::move(*permission_set_json)); + Json permission_set_json = parse_permission_set_to_json(and_rules); + permission_json.emplace("andRules", std::move(permission_set_json)); } else if (envoy_config_rbac_v3_Permission_has_or_rules(permission)) { + ValidationErrors::ScopedField field(errors, ".or_permission"); const auto* or_rules = envoy_config_rbac_v3_Permission_or_rules(permission); - auto permission_set_json = parse_permission_set_to_json(or_rules); - if (!permission_set_json.ok()) { - return permission_set_json; - } - permission_json.emplace("orRules", std::move(*permission_set_json)); + Json permission_set_json = parse_permission_set_to_json(or_rules); + permission_json.emplace("orRules", std::move(permission_set_json)); } else if (envoy_config_rbac_v3_Permission_has_any(permission)) { - permission_json.emplace("any", - envoy_config_rbac_v3_Permission_any(permission)); + permission_json.emplace( + "any", Json::FromBool(envoy_config_rbac_v3_Permission_any(permission))); } else if (envoy_config_rbac_v3_Permission_has_header(permission)) { - auto header_json = ParseHeaderMatcherToJson( - envoy_config_rbac_v3_Permission_header(permission)); - if (!header_json.ok()) { - return header_json; - } - permission_json.emplace("header", std::move(*header_json)); + ValidationErrors::ScopedField field(errors, ".header"); + Json header_json = ParseHeaderMatcherToJson( + envoy_config_rbac_v3_Permission_header(permission), errors); + permission_json.emplace("header", std::move(header_json)); } else if (envoy_config_rbac_v3_Permission_has_url_path(permission)) { - auto url_path_json = ParsePathMatcherToJson( - envoy_config_rbac_v3_Permission_url_path(permission)); - if (!url_path_json.ok()) { - return url_path_json; - } - permission_json.emplace("urlPath", std::move(*url_path_json)); + ValidationErrors::ScopedField field(errors, ".url_path"); + Json url_path_json = ParsePathMatcherToJson( + envoy_config_rbac_v3_Permission_url_path(permission), errors); + permission_json.emplace("urlPath", std::move(url_path_json)); } else if (envoy_config_rbac_v3_Permission_has_destination_ip(permission)) { permission_json.emplace( "destinationIp", @@ -261,94 +263,77 @@ absl::StatusOr<Json> ParsePermissionToJson( } else if (envoy_config_rbac_v3_Permission_has_destination_port(permission)) { permission_json.emplace( "destinationPort", - envoy_config_rbac_v3_Permission_destination_port(permission)); + Json::FromNumber( + envoy_config_rbac_v3_Permission_destination_port(permission))); } else if (envoy_config_rbac_v3_Permission_has_metadata(permission)) { permission_json.emplace( "metadata", ParseMetadataMatcherToJson( envoy_config_rbac_v3_Permission_metadata(permission))); } else if (envoy_config_rbac_v3_Permission_has_not_rule(permission)) { - auto not_rule_json = ParsePermissionToJson( - envoy_config_rbac_v3_Permission_not_rule(permission)); - if (!not_rule_json.ok()) { - return not_rule_json; - } - permission_json.emplace("notRule", std::move(*not_rule_json)); + ValidationErrors::ScopedField field(errors, ".not_rule"); + Json not_rule_json = ParsePermissionToJson( + envoy_config_rbac_v3_Permission_not_rule(permission), errors); + permission_json.emplace("notRule", std::move(not_rule_json)); } else if (envoy_config_rbac_v3_Permission_has_requested_server_name( permission)) { - auto requested_server_name_json = ParseStringMatcherToJson( - envoy_config_rbac_v3_Permission_requested_server_name(permission)); - if (!requested_server_name_json.ok()) { - return requested_server_name_json; - } + ValidationErrors::ScopedField field(errors, ".requested_server_name"); + Json requested_server_name_json = ParseStringMatcherToJson( + envoy_config_rbac_v3_Permission_requested_server_name(permission), + errors); permission_json.emplace("requestedServerName", - std::move(*requested_server_name_json)); + std::move(requested_server_name_json)); } else { - return absl::InvalidArgumentError("Permission: Invalid rule"); + errors->AddError("invalid rule"); } - return permission_json; + return Json::FromObject(std::move(permission_json)); } -absl::StatusOr<Json> ParsePrincipalToJson( - const envoy_config_rbac_v3_Principal* principal) { +Json ParsePrincipalToJson(const envoy_config_rbac_v3_Principal* principal, + ValidationErrors* errors) { Json::Object principal_json; // Helper function to parse Principal::Set to JSON. Used by `and_ids` and // `or_ids`. auto parse_principal_set_to_json = - [](const envoy_config_rbac_v3_Principal_Set* set) - -> absl::StatusOr<Json> { - Json::Object json; - std::vector<absl::Status> error_list; + [errors](const envoy_config_rbac_v3_Principal_Set* set) -> Json { Json::Array ids_json; size_t size; const envoy_config_rbac_v3_Principal* const* ids = envoy_config_rbac_v3_Principal_Set_ids(set, &size); for (size_t i = 0; i < size; ++i) { - auto principal_json = ParsePrincipalToJson(ids[i]); - if (!principal_json.ok()) { - error_list.push_back(principal_json.status()); - } else { - ids_json.emplace_back(std::move(*principal_json)); - } - } - if (!error_list.empty()) { - return StatusCreate(absl::StatusCode::kInvalidArgument, - "Error parsing Set", DEBUG_LOCATION, - std::move(error_list)); + ValidationErrors::ScopedField field(errors, + absl::StrCat(".ids[", i, "]")); + Json principal_json = ParsePrincipalToJson(ids[i], errors); + ids_json.emplace_back(std::move(principal_json)); } - return Json::Object({{"ids", std::move(ids_json)}}); + return Json::FromObject({{"ids", Json::FromArray(std::move(ids_json))}}); }; if (envoy_config_rbac_v3_Principal_has_and_ids(principal)) { + ValidationErrors::ScopedField field(errors, ".and_ids"); const auto* and_rules = envoy_config_rbac_v3_Principal_and_ids(principal); - auto principal_set_json = parse_principal_set_to_json(and_rules); - if (!principal_set_json.ok()) { - return principal_set_json; - } - principal_json.emplace("andIds", std::move(*principal_set_json)); + Json principal_set_json = parse_principal_set_to_json(and_rules); + principal_json.emplace("andIds", std::move(principal_set_json)); } else if (envoy_config_rbac_v3_Principal_has_or_ids(principal)) { + ValidationErrors::ScopedField field(errors, ".or_ids"); const auto* or_rules = envoy_config_rbac_v3_Principal_or_ids(principal); - auto principal_set_json = parse_principal_set_to_json(or_rules); - if (!principal_set_json.ok()) { - return principal_set_json; - } - principal_json.emplace("orIds", std::move(*principal_set_json)); + Json principal_set_json = parse_principal_set_to_json(or_rules); + principal_json.emplace("orIds", std::move(principal_set_json)); } else if (envoy_config_rbac_v3_Principal_has_any(principal)) { - principal_json.emplace("any", - envoy_config_rbac_v3_Principal_any(principal)); + principal_json.emplace( + "any", Json::FromBool(envoy_config_rbac_v3_Principal_any(principal))); } else if (envoy_config_rbac_v3_Principal_has_authenticated(principal)) { - auto* authenticated_json = - principal_json.emplace("authenticated", Json::Object()) - .first->second.mutable_object(); + Json::Object authenticated_json; const auto* principal_name = envoy_config_rbac_v3_Principal_Authenticated_principal_name( envoy_config_rbac_v3_Principal_authenticated(principal)); if (principal_name != nullptr) { - auto principal_name_json = ParseStringMatcherToJson(principal_name); - if (!principal_name_json.ok()) { - return principal_name_json; - } - authenticated_json->emplace("principalName", - std::move(*principal_name_json)); + ValidationErrors::ScopedField field(errors, + ".authenticated.principal_name"); + Json principal_name_json = + ParseStringMatcherToJson(principal_name, errors); + authenticated_json["principalName"] = std::move(principal_name_json); } + principal_json["authenticated"] = + Json::FromObject(std::move(authenticated_json)); } else if (envoy_config_rbac_v3_Principal_has_source_ip(principal)) { principal_json.emplace( "sourceIp", ParseCidrRangeToJson( @@ -363,95 +348,105 @@ absl::StatusOr<Json> ParsePrincipalToJson( "remoteIp", ParseCidrRangeToJson( envoy_config_rbac_v3_Principal_remote_ip(principal))); } else if (envoy_config_rbac_v3_Principal_has_header(principal)) { - auto header_json = ParseHeaderMatcherToJson( - envoy_config_rbac_v3_Principal_header(principal)); - if (!header_json.ok()) { - return header_json; - } - principal_json.emplace("header", std::move(*header_json)); + ValidationErrors::ScopedField field(errors, ".header"); + Json header_json = ParseHeaderMatcherToJson( + envoy_config_rbac_v3_Principal_header(principal), errors); + principal_json.emplace("header", std::move(header_json)); } else if (envoy_config_rbac_v3_Principal_has_url_path(principal)) { - auto url_path_json = ParsePathMatcherToJson( - envoy_config_rbac_v3_Principal_url_path(principal)); - if (!url_path_json.ok()) { - return url_path_json; - } - principal_json.emplace("urlPath", std::move(*url_path_json)); + ValidationErrors::ScopedField field(errors, ".url_path"); + Json url_path_json = ParsePathMatcherToJson( + envoy_config_rbac_v3_Principal_url_path(principal), errors); + principal_json.emplace("urlPath", std::move(url_path_json)); } else if (envoy_config_rbac_v3_Principal_has_metadata(principal)) { principal_json.emplace( "metadata", ParseMetadataMatcherToJson( envoy_config_rbac_v3_Principal_metadata(principal))); } else if (envoy_config_rbac_v3_Principal_has_not_id(principal)) { - auto not_id_json = - ParsePrincipalToJson(envoy_config_rbac_v3_Principal_not_id(principal)); - if (!not_id_json.ok()) { - return not_id_json; - } - principal_json.emplace("notId", std::move(*not_id_json)); + ValidationErrors::ScopedField field(errors, ".not_id"); + Json not_id_json = ParsePrincipalToJson( + envoy_config_rbac_v3_Principal_not_id(principal), errors); + principal_json.emplace("notId", std::move(not_id_json)); } else { - return absl::InvalidArgumentError("Principal: Invalid rule"); + errors->AddError("invalid rule"); } - return principal_json; + return Json::FromObject(std::move(principal_json)); } -absl::StatusOr<Json> ParsePolicyToJson( - const envoy_config_rbac_v3_Policy* policy) { +Json ParsePolicyToJson(const envoy_config_rbac_v3_Policy* policy, + ValidationErrors* errors) { Json::Object policy_json; - std::vector<absl::Status> error_list; size_t size; Json::Array permissions_json; const envoy_config_rbac_v3_Permission* const* permissions = envoy_config_rbac_v3_Policy_permissions(policy, &size); for (size_t i = 0; i < size; ++i) { - auto permission_json = ParsePermissionToJson(permissions[i]); - if (!permission_json.ok()) { - error_list.push_back(permission_json.status()); - } else { - permissions_json.emplace_back(std::move(*permission_json)); - } + ValidationErrors::ScopedField field(errors, + absl::StrCat(".permissions[", i, "]")); + Json permission_json = ParsePermissionToJson(permissions[i], errors); + permissions_json.emplace_back(std::move(permission_json)); } - policy_json.emplace("permissions", std::move(permissions_json)); + policy_json.emplace("permissions", + Json::FromArray(std::move(permissions_json))); Json::Array principals_json; const envoy_config_rbac_v3_Principal* const* principals = envoy_config_rbac_v3_Policy_principals(policy, &size); for (size_t i = 0; i < size; ++i) { - auto principal_json = ParsePrincipalToJson(principals[i]); - if (!principal_json.ok()) { - error_list.push_back(principal_json.status()); - } else { - principals_json.emplace_back(std::move(*principal_json)); - } + ValidationErrors::ScopedField field(errors, + absl::StrCat(".principals[", i, "]")); + Json principal_json = ParsePrincipalToJson(principals[i], errors); + principals_json.emplace_back(std::move(principal_json)); } - policy_json.emplace("principals", std::move(principals_json)); + policy_json.emplace("principals", + Json::FromArray(std::move(principals_json))); if (envoy_config_rbac_v3_Policy_has_condition(policy)) { - error_list.push_back( - absl::InvalidArgumentError("Policy: condition not supported")); + ValidationErrors::ScopedField field(errors, ".condition"); + errors->AddError("condition not supported"); } if (envoy_config_rbac_v3_Policy_has_checked_condition(policy)) { - error_list.push_back( - absl::InvalidArgumentError("Policy: checked condition not supported")); + ValidationErrors::ScopedField field(errors, ".checked_condition"); + errors->AddError("checked condition not supported"); } - if (!error_list.empty()) { - return StatusCreate(absl::StatusCode::kInvalidArgument, - "Error parsing Policy", DEBUG_LOCATION, - std::move(error_list)); + return Json::FromObject(std::move(policy_json)); +} + +Json ParseAuditLoggerConfigsToJson( + const XdsResourceType::DecodeContext& context, + const envoy_config_rbac_v3_RBAC_AuditLoggingOptions* audit_logging_options, + ValidationErrors* errors) { + Json::Array logger_configs_json; + size_t size; + const auto& registry = + static_cast<const GrpcXdsBootstrap&>(context.client->bootstrap()) + .audit_logger_registry(); + const envoy_config_rbac_v3_RBAC_AuditLoggingOptions_AuditLoggerConfig* const* + logger_configs = + envoy_config_rbac_v3_RBAC_AuditLoggingOptions_logger_configs( + audit_logging_options, &size); + for (size_t i = 0; i < size; ++i) { + ValidationErrors::ScopedField field( + errors, absl::StrCat(".logger_configs[", i, "]")); + logger_configs_json.emplace_back(registry.ConvertXdsAuditLoggerConfig( + context, logger_configs[i], errors)); } - return policy_json; + return Json::FromArray(logger_configs_json); } -absl::StatusOr<Json> ParseHttpRbacToJson( - const envoy_extensions_filters_http_rbac_v3_RBAC* rbac) { +Json ParseHttpRbacToJson(const XdsResourceType::DecodeContext& context, + const envoy_extensions_filters_http_rbac_v3_RBAC* rbac, + ValidationErrors* errors) { Json::Object rbac_json; - std::vector<absl::Status> error_list; const auto* rules = envoy_extensions_filters_http_rbac_v3_RBAC_rules(rbac); if (rules != nullptr) { + ValidationErrors::ScopedField field(errors, ".rules"); int action = envoy_config_rbac_v3_RBAC_action(rules); // Treat Log action as RBAC being absent if (action == envoy_config_rbac_v3_RBAC_LOG) { - return rbac_json; + return Json::FromObject({}); } Json::Object inner_rbac_json; - inner_rbac_json.emplace("action", envoy_config_rbac_v3_RBAC_action(rules)); - if (envoy_config_rbac_v3_RBAC_has_policies(rules)) { + inner_rbac_json.emplace( + "action", Json::FromNumber(envoy_config_rbac_v3_RBAC_action(rules))); + if (envoy_config_rbac_v3_RBAC_policies_size(rules) != 0) { Json::Object policies_object; size_t iter = kUpb_Map_Begin; while (true) { @@ -459,105 +454,138 @@ absl::StatusOr<Json> ParseHttpRbacToJson( if (entry == nullptr) { break; } - auto policy = ParsePolicyToJson( - envoy_config_rbac_v3_RBAC_PoliciesEntry_value(entry)); - if (!policy.ok()) { - error_list.push_back(StatusCreate( - absl::StatusCode::kInvalidArgument, - absl::StrFormat( - "RBAC PoliciesEntry key:%s", - UpbStringToStdString( - envoy_config_rbac_v3_RBAC_PoliciesEntry_key(entry))), - DEBUG_LOCATION, {policy.status()})); - } else { - policies_object.emplace( - UpbStringToStdString( - envoy_config_rbac_v3_RBAC_PoliciesEntry_key(entry)), - std::move(*policy)); - } + absl::string_view key = + UpbStringToAbsl(envoy_config_rbac_v3_RBAC_PoliciesEntry_key(entry)); + ValidationErrors::ScopedField field( + errors, absl::StrCat(".policies[", key, "]")); + Json policy = ParsePolicyToJson( + envoy_config_rbac_v3_RBAC_PoliciesEntry_value(entry), errors); + policies_object.emplace(std::string(key), std::move(policy)); } - inner_rbac_json.emplace("policies", std::move(policies_object)); + inner_rbac_json.emplace("policies", + Json::FromObject(std::move(policies_object))); } - rbac_json.emplace("rules", std::move(inner_rbac_json)); - } - if (!error_list.empty()) { - return StatusCreate(absl::StatusCode::kInvalidArgument, - "Error parsing RBAC", DEBUG_LOCATION, - std::move(error_list)); + // Flatten the nested messages defined in rbac.proto + if (XdsRbacAuditLoggingEnabled() && + envoy_config_rbac_v3_RBAC_has_audit_logging_options(rules)) { + ValidationErrors::ScopedField field(errors, ".audit_logging_options"); + const auto* audit_logging_options = + envoy_config_rbac_v3_RBAC_audit_logging_options(rules); + int32_t audit_condition = + envoy_config_rbac_v3_RBAC_AuditLoggingOptions_audit_condition( + audit_logging_options); + switch (audit_condition) { + case envoy_config_rbac_v3_RBAC_AuditLoggingOptions_NONE: + case envoy_config_rbac_v3_RBAC_AuditLoggingOptions_ON_DENY: + case envoy_config_rbac_v3_RBAC_AuditLoggingOptions_ON_ALLOW: + case envoy_config_rbac_v3_RBAC_AuditLoggingOptions_ON_DENY_AND_ALLOW: + inner_rbac_json.emplace("audit_condition", + Json::FromNumber(audit_condition)); + break; + default: + ValidationErrors::ScopedField field(errors, ".audit_condition"); + errors->AddError("invalid audit condition"); + } + if (envoy_config_rbac_v3_RBAC_AuditLoggingOptions_has_logger_configs( + audit_logging_options)) { + inner_rbac_json.emplace("audit_loggers", + ParseAuditLoggerConfigsToJson( + context, audit_logging_options, errors)); + } + } + rbac_json.emplace("rules", Json::FromObject(std::move(inner_rbac_json))); } - return rbac_json; + return Json::FromObject(std::move(rbac_json)); } } // namespace +absl::string_view XdsHttpRbacFilter::ConfigProtoName() const { + return "envoy.extensions.filters.http.rbac.v3.RBAC"; +} + +absl::string_view XdsHttpRbacFilter::OverrideConfigProtoName() const { + return "envoy.extensions.filters.http.rbac.v3.RBACPerRoute"; +} + void XdsHttpRbacFilter::PopulateSymtab(upb_DefPool* symtab) const { envoy_extensions_filters_http_rbac_v3_RBAC_getmsgdef(symtab); } -absl::StatusOr<XdsHttpFilterImpl::FilterConfig> -XdsHttpRbacFilter::GenerateFilterConfig(upb_StringView serialized_filter_config, - upb_Arena* arena) const { - absl::StatusOr<Json> rbac_json; +absl::optional<XdsHttpFilterImpl::FilterConfig> +XdsHttpRbacFilter::GenerateFilterConfig( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const { + absl::string_view* serialized_filter_config = + absl::get_if<absl::string_view>(&extension.value); + if (serialized_filter_config == nullptr) { + errors->AddError("could not parse HTTP RBAC filter config"); + return absl::nullopt; + } auto* rbac = envoy_extensions_filters_http_rbac_v3_RBAC_parse( - serialized_filter_config.data, serialized_filter_config.size, arena); + serialized_filter_config->data(), serialized_filter_config->size(), + context.arena); if (rbac == nullptr) { - return absl::InvalidArgumentError( - "could not parse HTTP RBAC filter config"); + errors->AddError("could not parse HTTP RBAC filter config"); + return absl::nullopt; } - rbac_json = ParseHttpRbacToJson(rbac); - if (!rbac_json.ok()) { - return rbac_json.status(); - } - return FilterConfig{kXdsHttpRbacFilterConfigName, std::move(*rbac_json)}; + return FilterConfig{ConfigProtoName(), + ParseHttpRbacToJson(context, rbac, errors)}; } -absl::StatusOr<XdsHttpFilterImpl::FilterConfig> +absl::optional<XdsHttpFilterImpl::FilterConfig> XdsHttpRbacFilter::GenerateFilterConfigOverride( - upb_StringView serialized_filter_config, upb_Arena* arena) const { + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const { + absl::string_view* serialized_filter_config = + absl::get_if<absl::string_view>(&extension.value); + if (serialized_filter_config == nullptr) { + errors->AddError("could not parse RBACPerRoute"); + return absl::nullopt; + } auto* rbac_per_route = envoy_extensions_filters_http_rbac_v3_RBACPerRoute_parse( - serialized_filter_config.data, serialized_filter_config.size, arena); + serialized_filter_config->data(), serialized_filter_config->size(), + context.arena); if (rbac_per_route == nullptr) { - return absl::InvalidArgumentError("could not parse RBACPerRoute"); + errors->AddError("could not parse RBACPerRoute"); + return absl::nullopt; } - absl::StatusOr<Json> rbac_json; + Json rbac_json; const auto* rbac = envoy_extensions_filters_http_rbac_v3_RBACPerRoute_rbac(rbac_per_route); if (rbac == nullptr) { - rbac_json = Json::Object(); + rbac_json = Json::FromObject({}); } else { - rbac_json = ParseHttpRbacToJson(rbac); - if (!rbac_json.ok()) { - return rbac_json.status(); - } + ValidationErrors::ScopedField field(errors, ".rbac"); + rbac_json = ParseHttpRbacToJson(context, rbac, errors); } - return FilterConfig{kXdsHttpRbacFilterConfigOverrideName, - std::move(*rbac_json)}; + return FilterConfig{OverrideConfigProtoName(), std::move(rbac_json)}; } const grpc_channel_filter* XdsHttpRbacFilter::channel_filter() const { return &RbacFilter::kFilterVtable; } -grpc_channel_args* XdsHttpRbacFilter::ModifyChannelArgs( - grpc_channel_args* args) const { - grpc_arg arg_to_add = grpc_channel_arg_integer_create( - const_cast<char*>(GRPC_ARG_PARSE_RBAC_METHOD_CONFIG), 1); - grpc_channel_args* new_args = - grpc_channel_args_copy_and_add(args, &arg_to_add, 1); - grpc_channel_args_destroy(args); - return new_args; +ChannelArgs XdsHttpRbacFilter::ModifyChannelArgs( + const ChannelArgs& args) const { + return args.Set(GRPC_ARG_PARSE_RBAC_METHOD_CONFIG, 1); } absl::StatusOr<XdsHttpFilterImpl::ServiceConfigJsonEntry> XdsHttpRbacFilter::GenerateServiceConfig( const FilterConfig& hcm_filter_config, - const FilterConfig* filter_config_override) const { - Json policy_json = filter_config_override != nullptr - ? filter_config_override->config - : hcm_filter_config.config; - // The policy JSON may be empty, that's allowed. - return ServiceConfigJsonEntry{"rbacPolicy", policy_json.Dump()}; + const FilterConfig* filter_config_override, + absl::string_view filter_name) const { + const Json& policy_json = filter_config_override != nullptr + ? filter_config_override->config + : hcm_filter_config.config; + auto json_object = policy_json.object(); + json_object.emplace("filter_name", + Json::FromString(std::string(filter_name))); + // The policy JSON may be empty other than the filter name, that's allowed. + return ServiceConfigJsonEntry{"rbacPolicy", + JsonDump(Json::FromObject(json_object))}; } } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_http_rbac_filter.h b/grpc/src/core/ext/xds/xds_http_rbac_filter.h index 785cd5db..74c71f3c 100644 --- a/grpc/src/core/ext/xds/xds_http_rbac_filter.h +++ b/grpc/src/core/ext/xds/xds_http_rbac_filter.h @@ -14,41 +14,46 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_HTTP_RBAC_FILTER_H -#define GRPC_CORE_EXT_XDS_XDS_HTTP_RBAC_FILTER_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_RBAC_FILTER_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_RBAC_FILTER_H #include <grpc/support/port_platform.h> +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "upb/reflection/def.h" + +#include "src/core/ext/xds/xds_common_types.h" #include "src/core/ext/xds/xds_http_filters.h" +#include "src/core/ext/xds/xds_resource_type.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_fwd.h" +#include "src/core/lib/gprpp/validation_errors.h" namespace grpc_core { -extern const char* kXdsHttpRbacFilterConfigName; -extern const char* kXdsHttpRbacFilterConfigOverrideName; - class XdsHttpRbacFilter : public XdsHttpFilterImpl { public: + absl::string_view ConfigProtoName() const override; + absl::string_view OverrideConfigProtoName() const override; void PopulateSymtab(upb_DefPool* symtab) const override; - - absl::StatusOr<FilterConfig> GenerateFilterConfig( - upb_StringView serialized_filter_config, upb_Arena* arena) const override; - - absl::StatusOr<FilterConfig> GenerateFilterConfigOverride( - upb_StringView serialized_filter_config, upb_Arena* arena) const override; - + absl::optional<FilterConfig> GenerateFilterConfig( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const override; + absl::optional<FilterConfig> GenerateFilterConfigOverride( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const override; const grpc_channel_filter* channel_filter() const override; - - grpc_channel_args* ModifyChannelArgs(grpc_channel_args* args) const override; - + ChannelArgs ModifyChannelArgs(const ChannelArgs& args) const override; absl::StatusOr<ServiceConfigJsonEntry> GenerateServiceConfig( const FilterConfig& hcm_filter_config, - const FilterConfig* filter_config_override) const override; - + const FilterConfig* filter_config_override, + absl::string_view filter_name) const override; bool IsSupportedOnClients() const override { return false; } - bool IsSupportedOnServers() const override { return true; } }; } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_HTTP_RBAC_FILTER_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_RBAC_FILTER_H diff --git a/grpc/src/core/ext/xds/xds_http_stateful_session_filter.cc b/grpc/src/core/ext/xds/xds_http_stateful_session_filter.cc new file mode 100644 index 00000000..85526a98 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_http_stateful_session_filter.cc @@ -0,0 +1,222 @@ +// +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include <grpc/support/port_platform.h> + +#include "src/core/ext/xds/xds_http_stateful_session_filter.h" + +#include <string> +#include <utility> + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/variant.h" +#include "envoy/config/core/v3/extension.upb.h" +#include "envoy/extensions/filters/http/stateful_session/v3/stateful_session.upb.h" +#include "envoy/extensions/filters/http/stateful_session/v3/stateful_session.upbdefs.h" +#include "envoy/extensions/http/stateful_session/cookie/v3/cookie.upb.h" +#include "envoy/extensions/http/stateful_session/cookie/v3/cookie.upbdefs.h" +#include "envoy/type/http/v3/cookie.upb.h" + +#include <grpc/support/json.h> + +#include "src/core/ext/filters/stateful_session/stateful_session_filter.h" +#include "src/core/ext/filters/stateful_session/stateful_session_service_config_parser.h" +#include "src/core/ext/xds/upb_utils.h" +#include "src/core/ext/xds/xds_common_types.h" +#include "src/core/ext/xds/xds_http_filters.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/gprpp/validation_errors.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_writer.h" + +namespace grpc_core { + +absl::string_view XdsHttpStatefulSessionFilter::ConfigProtoName() const { + return "envoy.extensions.filters.http.stateful_session.v3.StatefulSession"; +} + +absl::string_view XdsHttpStatefulSessionFilter::OverrideConfigProtoName() + const { + return "envoy.extensions.filters.http.stateful_session.v3" + ".StatefulSessionPerRoute"; +} + +void XdsHttpStatefulSessionFilter::PopulateSymtab(upb_DefPool* symtab) const { + envoy_extensions_filters_http_stateful_session_v3_StatefulSession_getmsgdef( + symtab); + envoy_extensions_filters_http_stateful_session_v3_StatefulSessionPerRoute_getmsgdef( + symtab); + envoy_extensions_http_stateful_session_cookie_v3_CookieBasedSessionState_getmsgdef( + symtab); +} + +namespace { + +Json::Object ValidateStatefulSession( + const XdsResourceType::DecodeContext& context, + const envoy_extensions_filters_http_stateful_session_v3_StatefulSession* + stateful_session, + ValidationErrors* errors) { + ValidationErrors::ScopedField field(errors, ".session_state"); + const auto* session_state = + envoy_extensions_filters_http_stateful_session_v3_StatefulSession_session_state( + stateful_session); + if (session_state == nullptr) { + errors->AddError("field not present"); + return {}; + } + ValidationErrors::ScopedField field2(errors, ".typed_config"); + const auto* typed_config = + envoy_config_core_v3_TypedExtensionConfig_typed_config(session_state); + auto extension = ExtractXdsExtension(context, typed_config, errors); + if (!extension.has_value()) return {}; + if (extension->type != + "envoy.extensions.http.stateful_session.cookie.v3" + ".CookieBasedSessionState") { + errors->AddError("unsupported session state type"); + return {}; + } + absl::string_view* serialized_session_state = + absl::get_if<absl::string_view>(&extension->value); + if (serialized_session_state == nullptr) { + errors->AddError("could not parse session state config"); + return {}; + } + auto* cookie_state = + envoy_extensions_http_stateful_session_cookie_v3_CookieBasedSessionState_parse( + serialized_session_state->data(), serialized_session_state->size(), + context.arena); + if (cookie_state == nullptr) { + errors->AddError("could not parse session state config"); + return {}; + } + ValidationErrors::ScopedField field3(errors, ".cookie"); + const auto* cookie = + envoy_extensions_http_stateful_session_cookie_v3_CookieBasedSessionState_cookie( + cookie_state); + if (cookie == nullptr) { + errors->AddError("field not present"); + return {}; + } + Json::Object cookie_config; + // name + std::string cookie_name = + UpbStringToStdString(envoy_type_http_v3_Cookie_name(cookie)); + if (cookie_name.empty()) { + ValidationErrors::ScopedField field(errors, ".name"); + errors->AddError("field not present"); + } + cookie_config["name"] = Json::FromString(std::move(cookie_name)); + // ttl + { + ValidationErrors::ScopedField field(errors, ".ttl"); + const auto* duration = envoy_type_http_v3_Cookie_ttl(cookie); + if (duration != nullptr) { + Duration ttl = ParseDuration(duration, errors); + cookie_config["ttl"] = Json::FromString(ttl.ToJsonString()); + } + } + // path + std::string path = + UpbStringToStdString(envoy_type_http_v3_Cookie_path(cookie)); + if (!path.empty()) cookie_config["path"] = Json::FromString(std::move(path)); + return cookie_config; +} + +} // namespace + +absl::optional<XdsHttpFilterImpl::FilterConfig> +XdsHttpStatefulSessionFilter::GenerateFilterConfig( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const { + absl::string_view* serialized_filter_config = + absl::get_if<absl::string_view>(&extension.value); + if (serialized_filter_config == nullptr) { + errors->AddError("could not parse stateful session filter config"); + return absl::nullopt; + } + auto* stateful_session = + envoy_extensions_filters_http_stateful_session_v3_StatefulSession_parse( + serialized_filter_config->data(), serialized_filter_config->size(), + context.arena); + if (stateful_session == nullptr) { + errors->AddError("could not parse stateful session filter config"); + return absl::nullopt; + } + return FilterConfig{ConfigProtoName(), + Json::FromObject(ValidateStatefulSession( + context, stateful_session, errors))}; +} + +absl::optional<XdsHttpFilterImpl::FilterConfig> +XdsHttpStatefulSessionFilter::GenerateFilterConfigOverride( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const { + absl::string_view* serialized_filter_config = + absl::get_if<absl::string_view>(&extension.value); + if (serialized_filter_config == nullptr) { + errors->AddError("could not parse stateful session filter override config"); + return absl::nullopt; + } + auto* stateful_session_per_route = + envoy_extensions_filters_http_stateful_session_v3_StatefulSessionPerRoute_parse( + serialized_filter_config->data(), serialized_filter_config->size(), + context.arena); + if (stateful_session_per_route == nullptr) { + errors->AddError("could not parse stateful session filter override config"); + return absl::nullopt; + } + Json::Object config; + if (!envoy_extensions_filters_http_stateful_session_v3_StatefulSessionPerRoute_disabled( + stateful_session_per_route)) { + ValidationErrors::ScopedField field(errors, ".stateful_session"); + const auto* stateful_session = + envoy_extensions_filters_http_stateful_session_v3_StatefulSessionPerRoute_stateful_session( + stateful_session_per_route); + if (stateful_session == nullptr) { + errors->AddError("field not present"); + } else { + config = ValidateStatefulSession(context, stateful_session, errors); + } + } + return FilterConfig{OverrideConfigProtoName(), + Json::FromObject(std::move(config))}; +} + +const grpc_channel_filter* XdsHttpStatefulSessionFilter::channel_filter() + const { + return &StatefulSessionFilter::kFilter; +} + +ChannelArgs XdsHttpStatefulSessionFilter::ModifyChannelArgs( + const ChannelArgs& args) const { + return args.Set(GRPC_ARG_PARSE_STATEFUL_SESSION_METHOD_CONFIG, 1); +} + +absl::StatusOr<XdsHttpFilterImpl::ServiceConfigJsonEntry> +XdsHttpStatefulSessionFilter::GenerateServiceConfig( + const FilterConfig& hcm_filter_config, + const FilterConfig* filter_config_override, + absl::string_view /*filter_name*/) const { + const Json& config = filter_config_override != nullptr + ? filter_config_override->config + : hcm_filter_config.config; + return ServiceConfigJsonEntry{"stateful_session", JsonDump(config)}; +} + +} // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_http_stateful_session_filter.h b/grpc/src/core/ext/xds/xds_http_stateful_session_filter.h new file mode 100644 index 00000000..71a2fab2 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_http_stateful_session_filter.h @@ -0,0 +1,59 @@ +// +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_STATEFUL_SESSION_FILTER_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_STATEFUL_SESSION_FILTER_H + +#include <grpc/support/port_platform.h> + +#include "absl/status/statusor.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "upb/reflection/def.h" + +#include "src/core/ext/xds/xds_common_types.h" +#include "src/core/ext/xds/xds_http_filters.h" +#include "src/core/ext/xds/xds_resource_type.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_fwd.h" +#include "src/core/lib/gprpp/validation_errors.h" + +namespace grpc_core { + +class XdsHttpStatefulSessionFilter : public XdsHttpFilterImpl { + public: + absl::string_view ConfigProtoName() const override; + absl::string_view OverrideConfigProtoName() const override; + void PopulateSymtab(upb_DefPool* symtab) const override; + absl::optional<FilterConfig> GenerateFilterConfig( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const override; + absl::optional<FilterConfig> GenerateFilterConfigOverride( + const XdsResourceType::DecodeContext& context, XdsExtension extension, + ValidationErrors* errors) const override; + const grpc_channel_filter* channel_filter() const override; + ChannelArgs ModifyChannelArgs(const ChannelArgs& args) const override; + absl::StatusOr<ServiceConfigJsonEntry> GenerateServiceConfig( + const FilterConfig& hcm_filter_config, + const FilterConfig* filter_config_override, + absl::string_view filter_name) const override; + bool IsSupportedOnClients() const override { return true; } + bool IsSupportedOnServers() const override { return false; } +}; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_HTTP_STATEFUL_SESSION_FILTER_H diff --git a/grpc/src/core/ext/xds/xds_lb_policy_registry.cc b/grpc/src/core/ext/xds/xds_lb_policy_registry.cc new file mode 100644 index 00000000..3cdff1af --- /dev/null +++ b/grpc/src/core/ext/xds/xds_lb_policy_registry.cc @@ -0,0 +1,335 @@ +// +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include <grpc/support/port_platform.h> + +#include "src/core/ext/xds/xds_lb_policy_registry.h" + +#include <stddef.h> +#include <stdint.h> + +#include <string> +#include <utility> + +#include "absl/strings/str_cat.h" +#include "absl/types/optional.h" +#include "absl/types/variant.h" +#include "envoy/config/core/v3/extension.upb.h" +#include "envoy/extensions/load_balancing_policies/client_side_weighted_round_robin/v3/client_side_weighted_round_robin.upb.h" +#include "envoy/extensions/load_balancing_policies/ring_hash/v3/ring_hash.upb.h" +#include "envoy/extensions/load_balancing_policies/wrr_locality/v3/wrr_locality.upb.h" +#include "google/protobuf/wrappers.upb.h" + +#include <grpc/support/json.h> + +#include "src/core/ext/xds/xds_common_types.h" +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/gprpp/validation_errors.h" +#include "src/core/lib/load_balancing/lb_policy_registry.h" + +namespace grpc_core { + +namespace { + +class RoundRobinLbPolicyConfigFactory + : public XdsLbPolicyRegistry::ConfigFactory { + public: + Json::Object ConvertXdsLbPolicyConfig( + const XdsLbPolicyRegistry* /*registry*/, + const XdsResourceType::DecodeContext& /*context*/, + absl::string_view /*configuration*/, ValidationErrors* /*errors*/, + int /*recursion_depth*/) override { + return Json::Object{{"round_robin", Json::FromObject({})}}; + } + + absl::string_view type() override { return Type(); } + + static absl::string_view Type() { + return "envoy.extensions.load_balancing_policies.round_robin.v3.RoundRobin"; + } +}; + +class ClientSideWeightedRoundRobinLbPolicyConfigFactory + : public XdsLbPolicyRegistry::ConfigFactory { + public: + Json::Object ConvertXdsLbPolicyConfig( + const XdsLbPolicyRegistry* /*registry*/, + const XdsResourceType::DecodeContext& context, + absl::string_view configuration, ValidationErrors* errors, + int /*recursion_depth*/) override { + const auto* resource = + envoy_extensions_load_balancing_policies_client_side_weighted_round_robin_v3_ClientSideWeightedRoundRobin_parse( + configuration.data(), configuration.size(), context.arena); + if (resource == nullptr) { + errors->AddError( + "can't decode ClientSideWeightedRoundRobin LB policy config"); + return {}; + } + Json::Object config; + // enable_oob_load_report + auto* enable_oob_load_report = + envoy_extensions_load_balancing_policies_client_side_weighted_round_robin_v3_ClientSideWeightedRoundRobin_enable_oob_load_report( + resource); + if (enable_oob_load_report != nullptr && + google_protobuf_BoolValue_value(enable_oob_load_report)) { + config["enableOobLoadReport"] = Json::FromBool(true); + } + // oob_reporting_period + auto* duration_proto = + envoy_extensions_load_balancing_policies_client_side_weighted_round_robin_v3_ClientSideWeightedRoundRobin_oob_reporting_period( + resource); + if (duration_proto != nullptr) { + ValidationErrors::ScopedField field(errors, ".oob_reporting_period"); + Duration duration = ParseDuration(duration_proto, errors); + config["oobReportingPeriod"] = Json::FromString(duration.ToJsonString()); + } + // blackout_period + duration_proto = + envoy_extensions_load_balancing_policies_client_side_weighted_round_robin_v3_ClientSideWeightedRoundRobin_blackout_period( + resource); + if (duration_proto != nullptr) { + ValidationErrors::ScopedField field(errors, ".blackout_period"); + Duration duration = ParseDuration(duration_proto, errors); + config["blackoutPeriod"] = Json::FromString(duration.ToJsonString()); + } + // weight_update_period + duration_proto = + envoy_extensions_load_balancing_policies_client_side_weighted_round_robin_v3_ClientSideWeightedRoundRobin_weight_update_period( + resource); + if (duration_proto != nullptr) { + ValidationErrors::ScopedField field(errors, ".weight_update_period"); + Duration duration = ParseDuration(duration_proto, errors); + config["weightUpdatePeriod"] = Json::FromString(duration.ToJsonString()); + } + // weight_expiration_period + duration_proto = + envoy_extensions_load_balancing_policies_client_side_weighted_round_robin_v3_ClientSideWeightedRoundRobin_weight_expiration_period( + resource); + if (duration_proto != nullptr) { + ValidationErrors::ScopedField field(errors, ".weight_expiration_period"); + Duration duration = ParseDuration(duration_proto, errors); + config["weightExpirationPeriod"] = + Json::FromString(duration.ToJsonString()); + } + // error_utilization_penalty + auto* error_utilization_penalty = + envoy_extensions_load_balancing_policies_client_side_weighted_round_robin_v3_ClientSideWeightedRoundRobin_error_utilization_penalty( + resource); + if (error_utilization_penalty != nullptr) { + ValidationErrors::ScopedField field(errors, ".error_utilization_penalty"); + const float value = + google_protobuf_FloatValue_value(error_utilization_penalty); + if (value < 0.0) { + errors->AddError("value must be non-negative"); + } + config["errorUtilizationPenalty"] = Json::FromNumber(value); + } + return Json::Object{ + {"weighted_round_robin", Json::FromObject(std::move(config))}}; + } + + absl::string_view type() override { return Type(); } + + static absl::string_view Type() { + return "envoy.extensions.load_balancing_policies.client_side_weighted_" + "round_robin.v3.ClientSideWeightedRoundRobin"; + } +}; + +class RingHashLbPolicyConfigFactory + : public XdsLbPolicyRegistry::ConfigFactory { + public: + Json::Object ConvertXdsLbPolicyConfig( + const XdsLbPolicyRegistry* /*registry*/, + const XdsResourceType::DecodeContext& context, + absl::string_view configuration, ValidationErrors* errors, + int /*recursion_depth*/) override { + const auto* resource = + envoy_extensions_load_balancing_policies_ring_hash_v3_RingHash_parse( + configuration.data(), configuration.size(), context.arena); + if (resource == nullptr) { + errors->AddError("can't decode RingHash LB policy config"); + return {}; + } + if (envoy_extensions_load_balancing_policies_ring_hash_v3_RingHash_hash_function( + resource) != + envoy_extensions_load_balancing_policies_ring_hash_v3_RingHash_XX_HASH && + envoy_extensions_load_balancing_policies_ring_hash_v3_RingHash_hash_function( + resource) != + envoy_extensions_load_balancing_policies_ring_hash_v3_RingHash_DEFAULT_HASH) { + ValidationErrors::ScopedField field(errors, ".hash_function"); + errors->AddError("unsupported value (must be XX_HASH)"); + } + uint64_t max_ring_size = 8388608; + const auto* uint64_value = + envoy_extensions_load_balancing_policies_ring_hash_v3_RingHash_maximum_ring_size( + resource); + if (uint64_value != nullptr) { + max_ring_size = google_protobuf_UInt64Value_value(uint64_value); + if (max_ring_size == 0 || max_ring_size > 8388608) { + ValidationErrors::ScopedField field(errors, ".maximum_ring_size"); + errors->AddError("value must be in the range [1, 8388608]"); + } + } + uint64_t min_ring_size = 1024; + uint64_value = + envoy_extensions_load_balancing_policies_ring_hash_v3_RingHash_minimum_ring_size( + resource); + if (uint64_value != nullptr) { + min_ring_size = google_protobuf_UInt64Value_value(uint64_value); + ValidationErrors::ScopedField field(errors, ".minimum_ring_size"); + if (min_ring_size == 0 || min_ring_size > 8388608) { + errors->AddError("value must be in the range [1, 8388608]"); + } + if (min_ring_size > max_ring_size) { + errors->AddError("cannot be greater than maximum_ring_size"); + } + } + return Json::Object{ + {"ring_hash_experimental", + Json::FromObject({ + {"minRingSize", Json::FromNumber(min_ring_size)}, + {"maxRingSize", Json::FromNumber(max_ring_size)}, + })}, + }; + } + + absl::string_view type() override { return Type(); } + + static absl::string_view Type() { + return "envoy.extensions.load_balancing_policies.ring_hash.v3.RingHash"; + } +}; + +class WrrLocalityLbPolicyConfigFactory + : public XdsLbPolicyRegistry::ConfigFactory { + public: + Json::Object ConvertXdsLbPolicyConfig( + const XdsLbPolicyRegistry* registry, + const XdsResourceType::DecodeContext& context, + absl::string_view configuration, ValidationErrors* errors, + int recursion_depth) override { + const auto* resource = + envoy_extensions_load_balancing_policies_wrr_locality_v3_WrrLocality_parse( + configuration.data(), configuration.size(), context.arena); + if (resource == nullptr) { + errors->AddError("can't decode WrrLocality LB policy config"); + return {}; + } + ValidationErrors::ScopedField field(errors, ".endpoint_picking_policy"); + const auto* endpoint_picking_policy = + envoy_extensions_load_balancing_policies_wrr_locality_v3_WrrLocality_endpoint_picking_policy( + resource); + if (endpoint_picking_policy == nullptr) { + errors->AddError("field not present"); + return {}; + } + auto child_policy = registry->ConvertXdsLbPolicyConfig( + context, endpoint_picking_policy, errors, recursion_depth + 1); + return Json::Object{ + {"xds_wrr_locality_experimental", + Json::FromObject( + {{"childPolicy", Json::FromArray(std::move(child_policy))}})}}; + } + + absl::string_view type() override { return Type(); } + + static absl::string_view Type() { + return "envoy.extensions.load_balancing_policies.wrr_locality.v3." + "WrrLocality"; + } +}; + +} // namespace + +// +// XdsLbPolicyRegistry +// + +XdsLbPolicyRegistry::XdsLbPolicyRegistry() { + policy_config_factories_.emplace( + RingHashLbPolicyConfigFactory::Type(), + std::make_unique<RingHashLbPolicyConfigFactory>()); + policy_config_factories_.emplace( + RoundRobinLbPolicyConfigFactory::Type(), + std::make_unique<RoundRobinLbPolicyConfigFactory>()); + policy_config_factories_.emplace( + ClientSideWeightedRoundRobinLbPolicyConfigFactory::Type(), + std::make_unique<ClientSideWeightedRoundRobinLbPolicyConfigFactory>()); + policy_config_factories_.emplace( + WrrLocalityLbPolicyConfigFactory::Type(), + std::make_unique<WrrLocalityLbPolicyConfigFactory>()); +} + +Json::Array XdsLbPolicyRegistry::ConvertXdsLbPolicyConfig( + const XdsResourceType::DecodeContext& context, + const envoy_config_cluster_v3_LoadBalancingPolicy* lb_policy, + ValidationErrors* errors, int recursion_depth) const { + constexpr int kMaxRecursionDepth = 16; + if (recursion_depth >= kMaxRecursionDepth) { + errors->AddError( + absl::StrCat("exceeded max recursion depth of ", kMaxRecursionDepth)); + return {}; + } + const size_t original_error_size = errors->size(); + size_t size = 0; + const auto* policies = + envoy_config_cluster_v3_LoadBalancingPolicy_policies(lb_policy, &size); + for (size_t i = 0; i < size; ++i) { + ValidationErrors::ScopedField field( + errors, absl::StrCat(".policies[", i, "].typed_extension_config")); + const auto* typed_extension_config = + envoy_config_cluster_v3_LoadBalancingPolicy_Policy_typed_extension_config( + policies[i]); + if (typed_extension_config == nullptr) { + errors->AddError("field not present"); + return {}; + } + ValidationErrors::ScopedField field2(errors, ".typed_config"); + const auto* typed_config = + envoy_config_core_v3_TypedExtensionConfig_typed_config( + typed_extension_config); + auto extension = ExtractXdsExtension(context, typed_config, errors); + if (!extension.has_value()) return {}; + // Check for registered LB policy type. + absl::string_view* serialized_value = + absl::get_if<absl::string_view>(&extension->value); + if (serialized_value != nullptr) { + auto config_factory_it = policy_config_factories_.find(extension->type); + if (config_factory_it != policy_config_factories_.end()) { + return Json::Array{Json::FromObject( + config_factory_it->second->ConvertXdsLbPolicyConfig( + this, context, *serialized_value, errors, recursion_depth))}; + } + } + // Check for custom LB policy type. + Json* json = absl::get_if<Json>(&extension->value); + if (json != nullptr && + CoreConfiguration::Get().lb_policy_registry().LoadBalancingPolicyExists( + extension->type, nullptr)) { + return Json::Array{ + Json::FromObject({{std::string(extension->type), std::move(*json)}})}; + } + // Unsupported type. Continue to next entry. + } + if (original_error_size == errors->size()) { + errors->AddError("no supported load balancing policy config found"); + } + return {}; +} + +} // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_lb_policy_registry.h b/grpc/src/core/ext/xds/xds_lb_policy_registry.h new file mode 100644 index 00000000..d797c9fb --- /dev/null +++ b/grpc/src/core/ext/xds/xds_lb_policy_registry.h @@ -0,0 +1,71 @@ +// +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_LB_POLICY_REGISTRY_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_LB_POLICY_REGISTRY_H + +#include <grpc/support/port_platform.h> + +#include <map> +#include <memory> + +#include "absl/strings/string_view.h" +#include "envoy/config/cluster/v3/cluster.upb.h" + +#include "src/core/ext/xds/xds_resource_type.h" +#include "src/core/lib/gprpp/validation_errors.h" +#include "src/core/lib/json/json.h" + +namespace grpc_core { + +// A registry that maintans a set of converters that are able to map xDS +// loadbalancing policy configurations to gRPC's JSON format. +class XdsLbPolicyRegistry { + public: + class ConfigFactory { + public: + virtual ~ConfigFactory() = default; + virtual Json::Object ConvertXdsLbPolicyConfig( + const XdsLbPolicyRegistry* registry, + const XdsResourceType::DecodeContext& context, + absl::string_view configuration, ValidationErrors* errors, + int recursion_depth) = 0; + virtual absl::string_view type() = 0; + }; + + XdsLbPolicyRegistry(); + + // Converts an xDS cluster load balancing policy message to gRPC's JSON + // format. An error is returned if none of the lb policies in the list are + // supported, or if a supported lb policy configuration conversion fails. \a + // recursion_depth indicates the current depth of the tree if lb_policy + // configuration recursively holds other lb policies. + Json::Array ConvertXdsLbPolicyConfig( + const XdsResourceType::DecodeContext& context, + const envoy_config_cluster_v3_LoadBalancingPolicy* lb_policy, + ValidationErrors* errors, int recursion_depth = 0) const; + + private: + // A map of config factories that goes from the type of the lb policy config + // to the config factory. + std::map<absl::string_view /* Owned by ConfigFactory */, + std::unique_ptr<ConfigFactory>> + policy_config_factories_; +}; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_LB_POLICY_REGISTRY_H diff --git a/grpc/src/core/ext/xds/xds_listener.cc b/grpc/src/core/ext/xds/xds_listener.cc index 14847e31..bdae5085 100644 --- a/grpc/src/core/ext/xds/xds_listener.cc +++ b/grpc/src/core/ext/xds/xds_listener.cc @@ -18,10 +18,17 @@ #include "src/core/ext/xds/xds_listener.h" +#include <stdint.h> + +#include <initializer_list> +#include <set> +#include <utility> + +#include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" -#include "absl/strings/str_split.h" #include "envoy/config/core/v3/address.upb.h" #include "envoy/config/core/v3/base.upb.h" #include "envoy/config/core/v3/config_source.upb.h" @@ -30,52 +37,51 @@ #include "envoy/config/listener/v3/listener.upb.h" #include "envoy/config/listener/v3/listener.upbdefs.h" #include "envoy/config/listener/v3/listener_components.upb.h" +#include "envoy/config/rbac/v3/rbac.upb.h" +#include "envoy/config/route/v3/route.upb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.h" +#include "envoy/extensions/transport_sockets/tls/v3/tls.upb.h" +#include "google/protobuf/any.upb.h" +#include "google/protobuf/duration.upb.h" #include "google/protobuf/wrappers.upb.h" -#include "upb/text_encode.h" -#include "upb/upb.h" -#include "upb/upb.hpp" +#include "upb/base/string_view.h" +#include "upb/text/encode.h" +#include <grpc/support/log.h> + +#include "src/core/ext/xds/upb_utils.h" #include "src/core/ext/xds/xds_common_types.h" +#include "src/core/ext/xds/xds_resource_type.h" #include "src/core/lib/address_utils/parse_address.h" #include "src/core/lib/address_utils/sockaddr_utils.h" +#include "src/core/lib/debug/trace.h" #include "src/core/lib/gprpp/host_port.h" +#include "src/core/lib/gprpp/match.h" +#include "src/core/lib/gprpp/validation_errors.h" #include "src/core/lib/iomgr/sockaddr.h" namespace grpc_core { // -// XdsListenerResource::DownstreamTlsContext -// - -std::string XdsListenerResource::DownstreamTlsContext::ToString() const { - return absl::StrFormat("common_tls_context=%s, require_client_certificate=%s", - common_tls_context.ToString(), - require_client_certificate ? "true" : "false"); -} - -bool XdsListenerResource::DownstreamTlsContext::Empty() const { - return common_tls_context.Empty(); -} - -// // XdsListenerResource::HttpConnectionManager // std::string XdsListenerResource::HttpConnectionManager::ToString() const { - absl::InlinedVector<std::string, 4> contents; - contents.push_back(absl::StrFormat( - "route_config_name=%s", - !route_config_name.empty() ? route_config_name.c_str() : "<inlined>")); - contents.push_back(absl::StrFormat("http_max_stream_duration=%s", - http_max_stream_duration.ToString())); - if (rds_update.has_value()) { - contents.push_back( - absl::StrFormat("rds_update=%s", rds_update->ToString())); - } + std::vector<std::string> contents; + contents.push_back(Match( + route_config, + [](const std::string& rds_name) { + return absl::StrCat("rds_name=", rds_name); + }, + [](const XdsRouteConfigResource& route_config) { + return absl::StrCat("route_config=", route_config.ToString()); + })); + contents.push_back(absl::StrCat("http_max_stream_duration=", + http_max_stream_duration.ToString())); if (!http_filters.empty()) { std::vector<std::string> filter_strings; + filter_strings.reserve(http_filters.size()); for (const auto& http_filter : http_filters) { filter_strings.push_back(http_filter.ToString()); } @@ -86,7 +92,7 @@ std::string XdsListenerResource::HttpConnectionManager::ToString() const { } // -// XdsListenerResource::HttpFilter +// XdsListenerResource::HttpConnectionManager::HttpFilter // std::string XdsListenerResource::HttpConnectionManager::HttpFilter::ToString() @@ -95,6 +101,20 @@ std::string XdsListenerResource::HttpConnectionManager::HttpFilter::ToString() } // +// XdsListenerResource::DownstreamTlsContext +// + +std::string XdsListenerResource::DownstreamTlsContext::ToString() const { + return absl::StrFormat("common_tls_context=%s, require_client_certificate=%s", + common_tls_context.ToString(), + require_client_certificate ? "true" : "false"); +} + +bool XdsListenerResource::DownstreamTlsContext::Empty() const { + return common_tls_context.Empty(); +} + +// // XdsListenerResource::FilterChainData // @@ -109,8 +129,10 @@ std::string XdsListenerResource::FilterChainData::ToString() const { // std::string XdsListenerResource::FilterChainMap::CidrRange::ToString() const { + auto addr_str = grpc_sockaddr_to_string(&address, false); return absl::StrCat( - "{address_prefix=", grpc_sockaddr_to_string(&address, false), + "{address_prefix=", + addr_str.ok() ? addr_str.value() : addr_str.status().ToString(), ", prefix_len=", prefix_len, "}"); } @@ -138,12 +160,13 @@ struct FilterChain { }; std::string FilterChain::FilterChainMatch::ToString() const { - absl::InlinedVector<std::string, 8> contents; + std::vector<std::string> contents; if (destination_port != 0) { contents.push_back(absl::StrCat("destination_port=", destination_port)); } if (!prefix_ranges.empty()) { std::vector<std::string> prefix_ranges_content; + prefix_ranges_content.reserve(prefix_ranges.size()); for (const auto& range : prefix_ranges) { prefix_ranges_content.push_back(range.ToString()); } @@ -159,6 +182,7 @@ std::string FilterChain::FilterChainMatch::ToString() const { } if (!source_prefix_ranges.empty()) { std::vector<std::string> source_prefix_ranges_content; + source_prefix_ranges_content.reserve(source_prefix_ranges.size()); for (const auto& range : source_prefix_ranges) { source_prefix_ranges_content.push_back(range.ToString()); } @@ -223,34 +247,44 @@ std::string XdsListenerResource::FilterChainMap::ToString() const { } // -// XdsListenerResource +// XdsListenerResource::TcpListener // -std::string XdsListenerResource::ToString() const { - absl::InlinedVector<std::string, 4> contents; - if (type == ListenerType::kTcpListener) { - contents.push_back(absl::StrCat("address=", address)); - contents.push_back( - absl::StrCat("filter_chain_map=", filter_chain_map.ToString())); - if (default_filter_chain.has_value()) { - contents.push_back(absl::StrCat("default_filter_chain=", - default_filter_chain->ToString())); - } - } else if (type == ListenerType::kHttpApiListener) { - contents.push_back(absl::StrFormat("http_connection_manager=%s", - http_connection_manager.ToString())); +std::string XdsListenerResource::TcpListener::ToString() const { + std::vector<std::string> contents; + contents.push_back(absl::StrCat("address=", address)); + contents.push_back( + absl::StrCat("filter_chain_map=", filter_chain_map.ToString())); + if (default_filter_chain.has_value()) { + contents.push_back(absl::StrCat("default_filter_chain=", + default_filter_chain->ToString())); } return absl::StrCat("{", absl::StrJoin(contents, ", "), "}"); } // +// XdsListenerResource +// + +std::string XdsListenerResource::ToString() const { + return Match( + listener, + [](const HttpConnectionManager& hcm) { + return absl::StrCat("{http_connection_manager=", hcm.ToString(), "}"); + }, + [](const TcpListener& tcp) { + return absl::StrCat("{tcp_listener=", tcp.ToString(), "}"); + }); +} + +// // XdsListenerResourceType // namespace { void MaybeLogHttpConnectionManager( - const XdsEncodingContext& context, + const XdsResourceType::DecodeContext& context, const envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager* http_connection_manager_config) { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && @@ -266,116 +300,144 @@ void MaybeLogHttpConnectionManager( } } -grpc_error_handle HttpConnectionManagerParse( - bool is_client, const XdsEncodingContext& context, - const envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager* - http_connection_manager_proto, - bool is_v2, - XdsListenerResource::HttpConnectionManager* http_connection_manager) { +XdsListenerResource::HttpConnectionManager HttpConnectionManagerParse( + bool is_client, const XdsResourceType::DecodeContext& context, + XdsExtension extension, ValidationErrors* errors) { + if (extension.type != + "envoy.extensions.filters.network.http_connection_manager.v3" + ".HttpConnectionManager") { + errors->AddError("unsupported filter type"); + return {}; + } + auto* serialized_hcm_config = + absl::get_if<absl::string_view>(&extension.value); + if (serialized_hcm_config == nullptr) { + errors->AddError("could not parse HttpConnectionManager config"); + return {}; + } + const auto* http_connection_manager_proto = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_parse( + serialized_hcm_config->data(), serialized_hcm_config->size(), + context.arena); + if (http_connection_manager_proto == nullptr) { + errors->AddError("could not parse HttpConnectionManager config"); + return {}; + } MaybeLogHttpConnectionManager(context, http_connection_manager_proto); - // NACK a non-zero `xff_num_trusted_hops` and a `non-empty - // original_ip_detection_extensions` as mentioned in + XdsListenerResource::HttpConnectionManager http_connection_manager; + // xff_num_trusted_hops -- must be zero as per // https://github.com/grpc/proposal/blob/master/A41-xds-rbac.md if (envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_xff_num_trusted_hops( http_connection_manager_proto) != 0) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "'xff_num_trusted_hops' must be zero"); + ValidationErrors::ScopedField field(errors, ".xff_num_trusted_hops"); + errors->AddError("must be zero"); } + // original_ip_detection_extensions -- must be empty as per + // https://github.com/grpc/proposal/blob/master/A41-xds-rbac.md if (envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_has_original_ip_detection_extensions( http_connection_manager_proto)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "'original_ip_detection_extensions' must be empty"); + ValidationErrors::ScopedField field(errors, + ".original_ip_detection_extensions"); + errors->AddError("must be empty"); } - // Obtain max_stream_duration from Http Protocol Options. + // common_http_protocol_options const envoy_config_core_v3_HttpProtocolOptions* options = envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_common_http_protocol_options( http_connection_manager_proto); if (options != nullptr) { + // max_stream_duration const google_protobuf_Duration* duration = envoy_config_core_v3_HttpProtocolOptions_max_stream_duration(options); if (duration != nullptr) { - http_connection_manager->http_max_stream_duration = - ParseDuration(duration); + ValidationErrors::ScopedField field( + errors, ".common_http_protocol_options.max_stream_duration"); + http_connection_manager.http_max_stream_duration = + ParseDuration(duration, errors); } } - // Parse filters. - if (!is_v2) { + // http_filters + { + ValidationErrors::ScopedField field(errors, ".http_filters"); + const auto& http_filter_registry = + static_cast<const GrpcXdsBootstrap&>(context.client->bootstrap()) + .http_filter_registry(); size_t num_filters = 0; const auto* http_filters = envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_http_filters( http_connection_manager_proto, &num_filters); std::set<absl::string_view> names_seen; + const size_t original_error_size = errors->size(); for (size_t i = 0; i < num_filters; ++i) { + ValidationErrors::ScopedField field(errors, absl::StrCat("[", i, "]")); const auto* http_filter = http_filters[i]; + // name absl::string_view name = UpbStringToAbsl( envoy_extensions_filters_network_http_connection_manager_v3_HttpFilter_name( http_filter)); - if (name.empty()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("empty filter name at index ", i)); - } - if (names_seen.find(name) != names_seen.end()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("duplicate HTTP filter name: ", name)); + { + ValidationErrors::ScopedField field(errors, ".name"); + if (name.empty()) { + errors->AddError("empty filter name"); + continue; + } + if (names_seen.find(name) != names_seen.end()) { + errors->AddError(absl::StrCat("duplicate HTTP filter name: ", name)); + continue; + } } names_seen.insert(name); + // is_optional const bool is_optional = envoy_extensions_filters_network_http_connection_manager_v3_HttpFilter_is_optional( http_filter); - const google_protobuf_Any* any = - envoy_extensions_filters_network_http_connection_manager_v3_HttpFilter_typed_config( - http_filter); - if (any == nullptr) { - if (is_optional) continue; - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("no filter config specified for filter name ", name)); - } - absl::string_view filter_type; - grpc_error_handle error = - ExtractExtensionTypeName(context, any, &filter_type); - if (error != GRPC_ERROR_NONE) return error; - const XdsHttpFilterImpl* filter_impl = - XdsHttpFilterRegistry::GetFilterForType(filter_type); - if (filter_impl == nullptr) { - if (is_optional) continue; - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("no filter registered for config type ", filter_type)); - } - if ((is_client && !filter_impl->IsSupportedOnClients()) || - (!is_client && !filter_impl->IsSupportedOnServers())) { - if (is_optional) continue; - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrFormat("Filter %s is not supported on %s", filter_type, - is_client ? "clients" : "servers")); - } - absl::StatusOr<XdsHttpFilterImpl::FilterConfig> filter_config = - filter_impl->GenerateFilterConfig(google_protobuf_Any_value(any), - context.arena); - if (!filter_config.ok()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat( - "filter config for type ", filter_type, - " failed to parse: ", StatusToString(filter_config.status()))); + // typed_config + { + ValidationErrors::ScopedField field(errors, ".typed_config"); + const google_protobuf_Any* typed_config = + envoy_extensions_filters_network_http_connection_manager_v3_HttpFilter_typed_config( + http_filter); + auto extension = ExtractXdsExtension(context, typed_config, errors); + if (!extension.has_value()) continue; + const XdsHttpFilterImpl* filter_impl = + http_filter_registry.GetFilterForType(extension->type); + if (filter_impl == nullptr) { + if (!is_optional) errors->AddError("unsupported filter type"); + continue; + } + if ((is_client && !filter_impl->IsSupportedOnClients()) || + (!is_client && !filter_impl->IsSupportedOnServers())) { + if (!is_optional) { + errors->AddError(absl::StrCat("filter is not supported on ", + is_client ? "clients" : "servers")); + } + continue; + } + absl::optional<XdsHttpFilterImpl::FilterConfig> filter_config = + filter_impl->GenerateFilterConfig(context, std::move(*extension), + errors); + if (filter_config.has_value()) { + http_connection_manager.http_filters.emplace_back( + XdsListenerResource::HttpConnectionManager::HttpFilter{ + std::string(name), std::move(*filter_config)}); + } } - http_connection_manager->http_filters.emplace_back( - XdsListenerResource::HttpConnectionManager::HttpFilter{ - std::string(name), std::move(*filter_config)}); } - if (http_connection_manager->http_filters.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Expected at least one HTTP filter"); + if (errors->size() == original_error_size && + http_connection_manager.http_filters.empty()) { + errors->AddError("expected at least one HTTP filter"); } // Make sure that the last filter is terminal and non-last filters are // non-terminal. Note that this check is being performed in a separate loop // to take care of the case where there are two terminal filters in the list // out of which only one gets added in the final list. - for (const auto& http_filter : http_connection_manager->http_filters) { + for (const auto& http_filter : http_connection_manager.http_filters) { const XdsHttpFilterImpl* filter_impl = - XdsHttpFilterRegistry::GetFilterForType( + http_filter_registry.GetFilterForType( http_filter.config.config_proto_type_name); - if (&http_filter != &http_connection_manager->http_filters.back()) { + if (&http_filter != &http_connection_manager.http_filters.back()) { // Filters before the last filter must not be terminal. if (filter_impl->IsTerminalFilter()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( + errors->AddError( absl::StrCat("terminal filter for config type ", http_filter.config.config_proto_type_name, " must be the last filter in the chain")); @@ -383,340 +445,351 @@ grpc_error_handle HttpConnectionManagerParse( } else { // The last filter must be terminal. if (!filter_impl->IsTerminalFilter()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( + errors->AddError( absl::StrCat("non-terminal filter for config type ", http_filter.config.config_proto_type_name, " is the last filter in the chain")); } } } + } + // Found inlined route_config. Parse it to find the cluster_name. + if (envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_has_route_config( + http_connection_manager_proto)) { + const envoy_config_route_v3_RouteConfiguration* route_config = + envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_route_config( + http_connection_manager_proto); + ValidationErrors::ScopedField field(errors, ".route_config"); + http_connection_manager.route_config = + XdsRouteConfigResource::Parse(context, route_config, errors); } else { - // If using a v2 config, we just hard-code a list containing only the - // router filter without actually looking at the config. This ensures - // that the right thing happens in the xds resolver without having - // to expose whether the resource we received was v2 or v3. - http_connection_manager->http_filters.emplace_back( - XdsListenerResource::HttpConnectionManager::HttpFilter{ - "router", {kXdsHttpRouterFilterConfigName, Json()}}); - } - // Guarding parsing of RouteConfig on the server side with the environmental - // variable since that's the first feature on the server side that will be - // using this. - if (is_client || XdsRbacEnabled()) { - // Found inlined route_config. Parse it to find the cluster_name. - if (envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_has_route_config( - http_connection_manager_proto)) { - const envoy_config_route_v3_RouteConfiguration* route_config = - envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_route_config( - http_connection_manager_proto); - XdsRouteConfigResource rds_update; - grpc_error_handle error = - XdsRouteConfigResource::Parse(context, route_config, &rds_update); - if (error != GRPC_ERROR_NONE) return error; - http_connection_manager->rds_update = std::move(rds_update); - return GRPC_ERROR_NONE; - } // Validate that RDS must be used to get the route_config dynamically. const envoy_extensions_filters_network_http_connection_manager_v3_Rds* rds = envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_rds( http_connection_manager_proto); if (rds == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "HttpConnectionManager neither has inlined route_config nor RDS."); - } - // Check that the ConfigSource specifies ADS. - const envoy_config_core_v3_ConfigSource* config_source = - envoy_extensions_filters_network_http_connection_manager_v3_Rds_config_source( - rds); - if (config_source == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "HttpConnectionManager missing config_source for RDS."); - } - if (!envoy_config_core_v3_ConfigSource_has_ads(config_source) && - !envoy_config_core_v3_ConfigSource_has_self(config_source)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "HttpConnectionManager ConfigSource for RDS does not specify ADS " - "or SELF."); + errors->AddError("neither route_config nor rds fields are present"); + } else { + // Get the route_config_name. + http_connection_manager.route_config = UpbStringToStdString( + envoy_extensions_filters_network_http_connection_manager_v3_Rds_route_config_name( + rds)); + // Check that the ConfigSource specifies ADS. + const envoy_config_core_v3_ConfigSource* config_source = + envoy_extensions_filters_network_http_connection_manager_v3_Rds_config_source( + rds); + ValidationErrors::ScopedField field(errors, ".rds.config_source"); + if (config_source == nullptr) { + errors->AddError("field not present"); + } else if (!envoy_config_core_v3_ConfigSource_has_ads(config_source) && + !envoy_config_core_v3_ConfigSource_has_self(config_source)) { + errors->AddError("ConfigSource does not specify ADS or SELF"); + } } - // Get the route_config_name. - http_connection_manager->route_config_name = UpbStringToStdString( - envoy_extensions_filters_network_http_connection_manager_v3_Rds_route_config_name( - rds)); } - return GRPC_ERROR_NONE; + return http_connection_manager; } -grpc_error_handle LdsResourceParseClient( - const XdsEncodingContext& context, - const envoy_config_listener_v3_ApiListener* api_listener, bool is_v2, - XdsListenerResource* lds_update) { - lds_update->type = XdsListenerResource::ListenerType::kHttpApiListener; - const upb_StringView encoded_api_listener = google_protobuf_Any_value( - envoy_config_listener_v3_ApiListener_api_listener(api_listener)); - const auto* http_connection_manager = - envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_parse( - encoded_api_listener.data, encoded_api_listener.size, context.arena); - if (http_connection_manager == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Could not parse HttpConnectionManager config from ApiListener"); - } - return HttpConnectionManagerParse(true /* is_client */, context, - http_connection_manager, is_v2, - &lds_update->http_connection_manager); +absl::StatusOr<XdsListenerResource> LdsResourceParseClient( + const XdsResourceType::DecodeContext& context, + const envoy_config_listener_v3_ApiListener* api_listener) { + XdsListenerResource lds_update; + ValidationErrors errors; + ValidationErrors::ScopedField field(&errors, "api_listener.api_listener"); + auto* api_listener_field = + envoy_config_listener_v3_ApiListener_api_listener(api_listener); + auto extension = ExtractXdsExtension(context, api_listener_field, &errors); + if (extension.has_value()) { + lds_update.listener = HttpConnectionManagerParse( + /*is_client=*/true, context, std::move(*extension), &errors); + } + if (!errors.ok()) { + return errors.status(absl::StatusCode::kInvalidArgument, + "errors validating ApiListener"); + } + return std::move(lds_update); } -grpc_error_handle DownstreamTlsContextParse( - const XdsEncodingContext& context, +XdsListenerResource::DownstreamTlsContext DownstreamTlsContextParse( + const XdsResourceType::DecodeContext& context, const envoy_config_core_v3_TransportSocket* transport_socket, - XdsListenerResource::DownstreamTlsContext* downstream_tls_context) { - absl::string_view name = UpbStringToAbsl( - envoy_config_core_v3_TransportSocket_name(transport_socket)); - if (name != "envoy.transport_sockets.tls") { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("Unrecognized transport socket: ", name)); - } - auto* typed_config = + ValidationErrors* errors) { + ValidationErrors::ScopedField field(errors, ".typed_config"); + const auto* typed_config = envoy_config_core_v3_TransportSocket_typed_config(transport_socket); - std::vector<grpc_error_handle> errors; - if (typed_config != nullptr) { - const upb_StringView encoded_downstream_tls_context = - google_protobuf_Any_value(typed_config); - auto* downstream_tls_context_proto = - envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_parse( - encoded_downstream_tls_context.data, - encoded_downstream_tls_context.size, context.arena); - if (downstream_tls_context_proto == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Can't decode downstream tls context."); - } - auto* common_tls_context = - envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_common_tls_context( - downstream_tls_context_proto); - if (common_tls_context != nullptr) { - grpc_error_handle error = - CommonTlsContext::Parse(context, common_tls_context, - &downstream_tls_context->common_tls_context); - if (error != GRPC_ERROR_NONE) errors.push_back(error); - } - auto* require_client_certificate = - envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_require_client_certificate( - downstream_tls_context_proto); - if (require_client_certificate != nullptr) { - downstream_tls_context->require_client_certificate = - google_protobuf_BoolValue_value(require_client_certificate); - } - auto* require_sni = - envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_require_sni( - downstream_tls_context_proto); - if (require_sni != nullptr && - google_protobuf_BoolValue_value(require_sni)) { - errors.push_back( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("require_sni: unsupported")); - } - if (envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_ocsp_staple_policy( - downstream_tls_context_proto) != - envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_LENIENT_STAPLING) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "ocsp_staple_policy: Only LENIENT_STAPLING supported")); + auto extension = ExtractXdsExtension(context, typed_config, errors); + if (!extension.has_value()) return {}; + if (extension->type != + "envoy.extensions.transport_sockets.tls.v3.DownstreamTlsContext") { + ValidationErrors::ScopedField field(errors, ".type_url"); + errors->AddError("unsupported transport socket type"); + return {}; + } + absl::string_view* serialized_downstream_tls_context = + absl::get_if<absl::string_view>(&extension->value); + if (serialized_downstream_tls_context == nullptr) { + errors->AddError("can't decode DownstreamTlsContext"); + return {}; + } + const auto* downstream_tls_context_proto = + envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_parse( + serialized_downstream_tls_context->data(), + serialized_downstream_tls_context->size(), context.arena); + if (downstream_tls_context_proto == nullptr) { + errors->AddError("can't decode DownstreamTlsContext"); + return {}; + } + XdsListenerResource::DownstreamTlsContext downstream_tls_context; + auto* common_tls_context = + envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_common_tls_context( + downstream_tls_context_proto); + if (common_tls_context != nullptr) { + ValidationErrors::ScopedField field(errors, ".common_tls_context"); + downstream_tls_context.common_tls_context = + CommonTlsContext::Parse(context, common_tls_context, errors); + // Note: We can't be more specific about the field name for this + // error, because we don't know which fields they were found in + // inside of CommonTlsContext, so we make the error message a bit + // more verbose to compensate. + if (!downstream_tls_context.common_tls_context + .certificate_validation_context.match_subject_alt_names.empty()) { + errors->AddError("match_subject_alt_names not supported on servers"); } } - if (downstream_tls_context->common_tls_context + // Note: We can't be more specific about the field name for this + // error, because we don't know which fields they were found in + // inside of CommonTlsContext, so we make the error message a bit + // more verbose to compensate. + if (downstream_tls_context.common_tls_context .tls_certificate_provider_instance.instance_name.empty()) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( + errors->AddError( "TLS configuration provided but no " - "tls_certificate_provider_instance found.")); - } - if (downstream_tls_context->require_client_certificate && - downstream_tls_context->common_tls_context.certificate_validation_context - .ca_certificate_provider_instance.instance_name.empty()) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "TLS configuration requires client certificates but no certificate " - "provider instance specified for validation.")); - } - if (!downstream_tls_context->common_tls_context.certificate_validation_context - .match_subject_alt_names.empty()) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "match_subject_alt_names not supported on servers")); - } - return GRPC_ERROR_CREATE_FROM_VECTOR("Error parsing DownstreamTlsContext", - &errors); + "tls_certificate_provider_instance found"); + } + auto* require_client_certificate = + envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_require_client_certificate( + downstream_tls_context_proto); + if (require_client_certificate != nullptr) { + downstream_tls_context.require_client_certificate = + google_protobuf_BoolValue_value(require_client_certificate); + if (downstream_tls_context.require_client_certificate && + downstream_tls_context.common_tls_context.certificate_validation_context + .ca_certificate_provider_instance.instance_name.empty()) { + ValidationErrors::ScopedField field(errors, + ".require_client_certificate"); + errors->AddError( + "client certificate required but no certificate " + "provider instance specified for validation"); + } + } + auto* require_sni = + envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_require_sni( + downstream_tls_context_proto); + if (require_sni != nullptr && google_protobuf_BoolValue_value(require_sni)) { + ValidationErrors::ScopedField field(errors, ".require_sni"); + errors->AddError("field unsupported"); + } + if (envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_ocsp_staple_policy( + downstream_tls_context_proto) != + envoy_extensions_transport_sockets_tls_v3_DownstreamTlsContext_LENIENT_STAPLING) { + ValidationErrors::ScopedField field(errors, ".ocsp_staple_policy"); + errors->AddError("value must be LENIENT_STAPLING"); + } + return downstream_tls_context; } -grpc_error_handle CidrRangeParse( +absl::optional<XdsListenerResource::FilterChainMap::CidrRange> CidrRangeParse( const envoy_config_core_v3_CidrRange* cidr_range_proto, - XdsListenerResource::FilterChainMap::CidrRange* cidr_range) { + ValidationErrors* errors) { + ValidationErrors::ScopedField field(errors, ".address_prefix"); + XdsListenerResource::FilterChainMap::CidrRange cidr_range; std::string address_prefix = UpbStringToStdString( envoy_config_core_v3_CidrRange_address_prefix(cidr_range_proto)); - grpc_error_handle error = - grpc_string_to_sockaddr(&cidr_range->address, address_prefix.c_str(), 0); - if (error != GRPC_ERROR_NONE) return error; - cidr_range->prefix_len = 0; + auto address = StringToSockaddr(address_prefix, /*port=*/0); + if (!address.ok()) { + errors->AddError(address.status().message()); + return absl::nullopt; + } + cidr_range.address = *address; + cidr_range.prefix_len = 0; auto* prefix_len_proto = envoy_config_core_v3_CidrRange_prefix_len(cidr_range_proto); if (prefix_len_proto != nullptr) { - cidr_range->prefix_len = std::min( + cidr_range.prefix_len = std::min( google_protobuf_UInt32Value_value(prefix_len_proto), - (reinterpret_cast<const grpc_sockaddr*>(cidr_range->address.addr)) + (reinterpret_cast<const grpc_sockaddr*>(cidr_range.address.addr)) ->sa_family == GRPC_AF_INET - ? uint32_t(32) - : uint32_t(128)); + ? uint32_t{32} + : uint32_t{128}); } // Normalize the network address by masking it with prefix_len - grpc_sockaddr_mask_bits(&cidr_range->address, cidr_range->prefix_len); - return GRPC_ERROR_NONE; + grpc_sockaddr_mask_bits(&cidr_range.address, cidr_range.prefix_len); + return cidr_range; } -grpc_error_handle FilterChainMatchParse( +absl::optional<FilterChain::FilterChainMatch> FilterChainMatchParse( const envoy_config_listener_v3_FilterChainMatch* filter_chain_match_proto, - FilterChain::FilterChainMatch* filter_chain_match) { + ValidationErrors* errors) { + FilterChain::FilterChainMatch filter_chain_match; + const size_t original_error_size = errors->size(); + // destination_port auto* destination_port = envoy_config_listener_v3_FilterChainMatch_destination_port( filter_chain_match_proto); if (destination_port != nullptr) { - filter_chain_match->destination_port = + filter_chain_match.destination_port = google_protobuf_UInt32Value_value(destination_port); } + // prefix_ranges size_t size = 0; auto* prefix_ranges = envoy_config_listener_v3_FilterChainMatch_prefix_ranges( filter_chain_match_proto, &size); - filter_chain_match->prefix_ranges.reserve(size); + filter_chain_match.prefix_ranges.reserve(size); for (size_t i = 0; i < size; i++) { - XdsListenerResource::FilterChainMap::CidrRange cidr_range; - grpc_error_handle error = CidrRangeParse(prefix_ranges[i], &cidr_range); - if (error != GRPC_ERROR_NONE) return error; - filter_chain_match->prefix_ranges.push_back(cidr_range); + ValidationErrors::ScopedField field( + errors, absl::StrCat(".prefix_ranges[", i, "]")); + auto cidr_range = CidrRangeParse(prefix_ranges[i], errors); + if (cidr_range.has_value()) { + filter_chain_match.prefix_ranges.push_back(*cidr_range); + } } - filter_chain_match->source_type = + // source_type + filter_chain_match.source_type = static_cast<XdsListenerResource::FilterChainMap::ConnectionSourceType>( envoy_config_listener_v3_FilterChainMatch_source_type( filter_chain_match_proto)); + // source_prefix_ranges auto* source_prefix_ranges = envoy_config_listener_v3_FilterChainMatch_source_prefix_ranges( filter_chain_match_proto, &size); - filter_chain_match->source_prefix_ranges.reserve(size); + filter_chain_match.source_prefix_ranges.reserve(size); for (size_t i = 0; i < size; i++) { - XdsListenerResource::FilterChainMap::CidrRange cidr_range; - grpc_error_handle error = - CidrRangeParse(source_prefix_ranges[i], &cidr_range); - if (error != GRPC_ERROR_NONE) return error; - filter_chain_match->source_prefix_ranges.push_back(cidr_range); + ValidationErrors::ScopedField field( + errors, absl::StrCat(".source_prefix_ranges[", i, "]")); + auto cidr_range = CidrRangeParse(source_prefix_ranges[i], errors); + if (cidr_range.has_value()) { + filter_chain_match.source_prefix_ranges.push_back(*cidr_range); + } } + // source_ports auto* source_ports = envoy_config_listener_v3_FilterChainMatch_source_ports( filter_chain_match_proto, &size); - filter_chain_match->source_ports.reserve(size); + filter_chain_match.source_ports.reserve(size); for (size_t i = 0; i < size; i++) { - filter_chain_match->source_ports.push_back(source_ports[i]); + filter_chain_match.source_ports.push_back(source_ports[i]); } + // server_names auto* server_names = envoy_config_listener_v3_FilterChainMatch_server_names( filter_chain_match_proto, &size); for (size_t i = 0; i < size; i++) { - filter_chain_match->server_names.push_back( + filter_chain_match.server_names.push_back( UpbStringToStdString(server_names[i])); } - filter_chain_match->transport_protocol = UpbStringToStdString( + // transport_protocol + filter_chain_match.transport_protocol = UpbStringToStdString( envoy_config_listener_v3_FilterChainMatch_transport_protocol( filter_chain_match_proto)); + // application_protocols auto* application_protocols = envoy_config_listener_v3_FilterChainMatch_application_protocols( filter_chain_match_proto, &size); for (size_t i = 0; i < size; i++) { - filter_chain_match->application_protocols.push_back( + filter_chain_match.application_protocols.push_back( UpbStringToStdString(application_protocols[i])); } - return GRPC_ERROR_NONE; + // Return result. + if (errors->size() != original_error_size) return absl::nullopt; + return filter_chain_match; } -grpc_error_handle FilterChainParse( - const XdsEncodingContext& context, - const envoy_config_listener_v3_FilterChain* filter_chain_proto, bool is_v2, - FilterChain* filter_chain) { - std::vector<grpc_error_handle> errors; +absl::optional<FilterChain> FilterChainParse( + const XdsResourceType::DecodeContext& context, + const envoy_config_listener_v3_FilterChain* filter_chain_proto, + ValidationErrors* errors) { + FilterChain filter_chain; + const size_t original_error_size = errors->size(); + // filter_chain_match auto* filter_chain_match = envoy_config_listener_v3_FilterChain_filter_chain_match( filter_chain_proto); if (filter_chain_match != nullptr) { - grpc_error_handle error = FilterChainMatchParse( - filter_chain_match, &filter_chain->filter_chain_match); - if (error != GRPC_ERROR_NONE) errors.push_back(error); + ValidationErrors::ScopedField field(errors, ".filter_chain_match"); + auto match = FilterChainMatchParse(filter_chain_match, errors); + if (match.has_value()) { + filter_chain.filter_chain_match = std::move(*match); + } } - filter_chain->filter_chain_data = - std::make_shared<XdsListenerResource::FilterChainData>(); - // Parse the filters list. Currently we only support HttpConnectionManager. - size_t size = 0; - auto* filters = - envoy_config_listener_v3_FilterChain_filters(filter_chain_proto, &size); - if (size != 1) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "FilterChain should have exactly one filter: HttpConnectionManager; no " - "other filter is supported at the moment")); - } else { - auto* typed_config = - envoy_config_listener_v3_Filter_typed_config(filters[0]); - if (typed_config == nullptr) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "No typed_config found in filter.")); - } else { - absl::string_view type_url = - UpbStringToAbsl(google_protobuf_Any_type_url(typed_config)); - if (type_url != - "type.googleapis.com/" - "envoy.extensions.filters.network.http_connection_manager.v3." - "HttpConnectionManager") { - errors.push_back(GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("Unsupported filter type ", type_url))); - } else { - const upb_StringView encoded_http_connection_manager = - google_protobuf_Any_value(typed_config); - const auto* http_connection_manager = - envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_parse( - encoded_http_connection_manager.data, - encoded_http_connection_manager.size, context.arena); - if (http_connection_manager == nullptr) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Could not parse HttpConnectionManager config from filter " - "typed_config")); - } else { - grpc_error_handle error = HttpConnectionManagerParse( - false /* is_client */, context, http_connection_manager, is_v2, - &filter_chain->filter_chain_data->http_connection_manager); - if (error != GRPC_ERROR_NONE) errors.push_back(error); - } + // filters + { + ValidationErrors::ScopedField field(errors, ".filters"); + filter_chain.filter_chain_data = + std::make_shared<XdsListenerResource::FilterChainData>(); + size_t size = 0; + auto* filters = + envoy_config_listener_v3_FilterChain_filters(filter_chain_proto, &size); + if (size != 1) { + errors->AddError( + "must have exactly one filter (HttpConnectionManager -- " + "no other filter is supported at the moment)"); + } + // entries in filters list + for (size_t i = 0; i < size; ++i) { + ValidationErrors::ScopedField field( + errors, absl::StrCat("[", i, "].typed_config")); + auto* typed_config = + envoy_config_listener_v3_Filter_typed_config(filters[i]); + auto extension = ExtractXdsExtension(context, typed_config, errors); + if (extension.has_value()) { + filter_chain.filter_chain_data->http_connection_manager = + HttpConnectionManagerParse(/*is_client=*/false, context, + std::move(*extension), errors); } } } + // transport_socket auto* transport_socket = envoy_config_listener_v3_FilterChain_transport_socket(filter_chain_proto); if (transport_socket != nullptr) { - grpc_error_handle error = DownstreamTlsContextParse( - context, transport_socket, - &filter_chain->filter_chain_data->downstream_tls_context); - if (error != GRPC_ERROR_NONE) errors.push_back(error); + ValidationErrors::ScopedField field(errors, ".transport_socket"); + filter_chain.filter_chain_data->downstream_tls_context = + DownstreamTlsContextParse(context, transport_socket, errors); } - return GRPC_ERROR_CREATE_FROM_VECTOR("Error parsing FilterChain", &errors); + // Return result. + if (errors->size() != original_error_size) return absl::nullopt; + return filter_chain; } -grpc_error_handle AddressParse( - const envoy_config_core_v3_Address* address_proto, std::string* address) { +absl::optional<std::string> AddressParse( + const envoy_config_core_v3_Address* address_proto, + ValidationErrors* errors) { + if (address_proto == nullptr) { + errors->AddError("field not present"); + return absl::nullopt; + } + ValidationErrors::ScopedField field(errors, ".socket_address"); const auto* socket_address = envoy_config_core_v3_Address_socket_address(address_proto); if (socket_address == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Address does not have socket_address"); + errors->AddError("field not present"); + return absl::nullopt; } - if (envoy_config_core_v3_SocketAddress_protocol(socket_address) != - envoy_config_core_v3_SocketAddress_TCP) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "SocketAddress protocol is not TCP"); + { + ValidationErrors::ScopedField field(errors, ".protocol"); + if (envoy_config_core_v3_SocketAddress_protocol(socket_address) != + envoy_config_core_v3_SocketAddress_TCP) { + errors->AddError("value must be TCP"); + } } + ValidationErrors::ScopedField field2(errors, ".port_value"); uint32_t port = envoy_config_core_v3_SocketAddress_port_value(socket_address); if (port > 65535) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Invalid port"); + errors->AddError("invalid port"); + return absl::nullopt; } - *address = JoinHostPort( + return JoinHostPort( UpbStringToAbsl( envoy_config_core_v3_SocketAddress_address(socket_address)), port); - return GRPC_ERROR_NONE; } // An intermediate map for filter chains that we create to validate the list of @@ -735,97 +808,103 @@ struct InternalFilterChainMap { DestinationIpMap destination_ip_map; }; -grpc_error_handle AddFilterChainDataForSourcePort( - const FilterChain& filter_chain, +void AddFilterChainDataForSourcePort( + const FilterChain& filter_chain, uint32_t port, XdsListenerResource::FilterChainMap::SourcePortsMap* ports_map, - uint32_t port) { + ValidationErrors* errors) { auto insert_result = ports_map->emplace( port, XdsListenerResource::FilterChainMap::FilterChainDataSharedPtr{ filter_chain.filter_chain_data}); if (!insert_result.second) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat( - "Duplicate matching rules detected when adding filter chain: ", + errors->AddError(absl::StrCat( + "duplicate matching rules detected when adding filter chain: ", filter_chain.filter_chain_match.ToString())); } - return GRPC_ERROR_NONE; } -grpc_error_handle AddFilterChainDataForSourcePorts( +void AddFilterChainDataForSourcePorts( const FilterChain& filter_chain, - XdsListenerResource::FilterChainMap::SourcePortsMap* ports_map) { + XdsListenerResource::FilterChainMap::SourcePortsMap* ports_map, + ValidationErrors* errors) { if (filter_chain.filter_chain_match.source_ports.empty()) { - return AddFilterChainDataForSourcePort(filter_chain, ports_map, 0); + AddFilterChainDataForSourcePort(filter_chain, 0, ports_map, errors); } else { for (uint32_t port : filter_chain.filter_chain_match.source_ports) { - grpc_error_handle error = - AddFilterChainDataForSourcePort(filter_chain, ports_map, port); - if (error != GRPC_ERROR_NONE) return error; + AddFilterChainDataForSourcePort(filter_chain, port, ports_map, errors); } } - return GRPC_ERROR_NONE; } -grpc_error_handle AddFilterChainDataForSourceIpRange( +void AddFilterChainDataForSourceIpRange( const FilterChain& filter_chain, - InternalFilterChainMap::SourceIpMap* source_ip_map) { + InternalFilterChainMap::SourceIpMap* source_ip_map, + ValidationErrors* errors) { if (filter_chain.filter_chain_match.source_prefix_ranges.empty()) { auto insert_result = source_ip_map->emplace( "", XdsListenerResource::FilterChainMap::SourceIp()); - return AddFilterChainDataForSourcePorts( - filter_chain, &insert_result.first->second.ports_map); + AddFilterChainDataForSourcePorts( + filter_chain, &insert_result.first->second.ports_map, errors); } else { for (const auto& prefix_range : filter_chain.filter_chain_match.source_prefix_ranges) { + auto addr_str = grpc_sockaddr_to_string(&prefix_range.address, false); + if (!addr_str.ok()) { + errors->AddError(absl::StrCat( + "error parsing source IP sockaddr (should not happen): ", + addr_str.status().message())); + continue; + } auto insert_result = source_ip_map->emplace( - absl::StrCat(grpc_sockaddr_to_string(&prefix_range.address, false), - "/", prefix_range.prefix_len), + absl::StrCat(*addr_str, "/", prefix_range.prefix_len), XdsListenerResource::FilterChainMap::SourceIp()); if (insert_result.second) { insert_result.first->second.prefix_range.emplace(prefix_range); } - grpc_error_handle error = AddFilterChainDataForSourcePorts( - filter_chain, &insert_result.first->second.ports_map); - if (error != GRPC_ERROR_NONE) return error; + AddFilterChainDataForSourcePorts( + filter_chain, &insert_result.first->second.ports_map, errors); } } - return GRPC_ERROR_NONE; } -grpc_error_handle AddFilterChainDataForSourceType( +void AddFilterChainDataForSourceType( const FilterChain& filter_chain, - InternalFilterChainMap::DestinationIp* destination_ip) { + InternalFilterChainMap::DestinationIp* destination_ip, + ValidationErrors* errors) { GPR_ASSERT(static_cast<unsigned int>( filter_chain.filter_chain_match.source_type) < 3); - return AddFilterChainDataForSourceIpRange( - filter_chain, &destination_ip->source_types_array[static_cast<int>( - filter_chain.filter_chain_match.source_type)]); + AddFilterChainDataForSourceIpRange( + filter_chain, + &destination_ip->source_types_array[static_cast<int>( + filter_chain.filter_chain_match.source_type)], + errors); } -grpc_error_handle AddFilterChainDataForApplicationProtocols( +void AddFilterChainDataForApplicationProtocols( const FilterChain& filter_chain, - InternalFilterChainMap::DestinationIp* destination_ip) { + InternalFilterChainMap::DestinationIp* destination_ip, + ValidationErrors* errors) { // Only allow filter chains that do not mention application protocols - if (!filter_chain.filter_chain_match.application_protocols.empty()) { - return GRPC_ERROR_NONE; + if (filter_chain.filter_chain_match.application_protocols.empty()) { + AddFilterChainDataForSourceType(filter_chain, destination_ip, errors); } - return AddFilterChainDataForSourceType(filter_chain, destination_ip); } -grpc_error_handle AddFilterChainDataForTransportProtocol( +void AddFilterChainDataForTransportProtocol( const FilterChain& filter_chain, - InternalFilterChainMap::DestinationIp* destination_ip) { + InternalFilterChainMap::DestinationIp* destination_ip, + ValidationErrors* errors) { const std::string& transport_protocol = filter_chain.filter_chain_match.transport_protocol; // Only allow filter chains with no transport protocol or "raw_buffer" if (!transport_protocol.empty() && transport_protocol != "raw_buffer") { - return GRPC_ERROR_NONE; + return; } // If for this configuration, we've already seen filter chains that mention // the transport protocol as "raw_buffer", we will never match filter chains // that do not mention it. if (destination_ip->transport_protocol_raw_buffer_provided && transport_protocol.empty()) { - return GRPC_ERROR_NONE; + return; } if (!transport_protocol.empty() && !destination_ip->transport_protocol_raw_buffer_provided) { @@ -835,44 +914,50 @@ grpc_error_handle AddFilterChainDataForTransportProtocol( destination_ip->source_types_array = InternalFilterChainMap::ConnectionSourceTypesArray(); } - return AddFilterChainDataForApplicationProtocols(filter_chain, - destination_ip); + AddFilterChainDataForApplicationProtocols(filter_chain, destination_ip, + errors); } -grpc_error_handle AddFilterChainDataForServerNames( +void AddFilterChainDataForServerNames( const FilterChain& filter_chain, - InternalFilterChainMap::DestinationIp* destination_ip) { + InternalFilterChainMap::DestinationIp* destination_ip, + ValidationErrors* errors) { // Don't continue adding filter chains with server names mentioned - if (!filter_chain.filter_chain_match.server_names.empty()) { - return GRPC_ERROR_NONE; + if (filter_chain.filter_chain_match.server_names.empty()) { + AddFilterChainDataForTransportProtocol(filter_chain, destination_ip, + errors); } - return AddFilterChainDataForTransportProtocol(filter_chain, destination_ip); } -grpc_error_handle AddFilterChainDataForDestinationIpRange( +void AddFilterChainDataForDestinationIpRange( const FilterChain& filter_chain, - InternalFilterChainMap::DestinationIpMap* destination_ip_map) { + InternalFilterChainMap::DestinationIpMap* destination_ip_map, + ValidationErrors* errors) { if (filter_chain.filter_chain_match.prefix_ranges.empty()) { auto insert_result = destination_ip_map->emplace( "", InternalFilterChainMap::DestinationIp()); - return AddFilterChainDataForServerNames(filter_chain, - &insert_result.first->second); + AddFilterChainDataForServerNames(filter_chain, &insert_result.first->second, + errors); } else { for (const auto& prefix_range : filter_chain.filter_chain_match.prefix_ranges) { + auto addr_str = grpc_sockaddr_to_string(&prefix_range.address, false); + if (!addr_str.ok()) { + errors->AddError(absl::StrCat( + "error parsing destination IP sockaddr (should not happen): ", + addr_str.status().message())); + continue; + } auto insert_result = destination_ip_map->emplace( - absl::StrCat(grpc_sockaddr_to_string(&prefix_range.address, false), - "/", prefix_range.prefix_len), + absl::StrCat(*addr_str, "/", prefix_range.prefix_len), InternalFilterChainMap::DestinationIp()); if (insert_result.second) { insert_result.first->second.prefix_range.emplace(prefix_range); } - grpc_error_handle error = AddFilterChainDataForServerNames( - filter_chain, &insert_result.first->second); - if (error != GRPC_ERROR_NONE) return error; + AddFilterChainDataForServerNames(filter_chain, + &insert_result.first->second, errors); } } - return GRPC_ERROR_NONE; } XdsListenerResource::FilterChainMap BuildFromInternalFilterChainMap( @@ -894,99 +979,113 @@ XdsListenerResource::FilterChainMap BuildFromInternalFilterChainMap( return filter_chain_map; } -grpc_error_handle BuildFilterChainMap( - const std::vector<FilterChain>& filter_chains, - XdsListenerResource::FilterChainMap* filter_chain_map) { +XdsListenerResource::FilterChainMap BuildFilterChainMap( + const std::vector<FilterChain>& filter_chains, ValidationErrors* errors) { InternalFilterChainMap internal_filter_chain_map; for (const auto& filter_chain : filter_chains) { // Discard filter chain entries that specify destination port if (filter_chain.filter_chain_match.destination_port != 0) continue; - grpc_error_handle error = AddFilterChainDataForDestinationIpRange( - filter_chain, &internal_filter_chain_map.destination_ip_map); - if (error != GRPC_ERROR_NONE) return error; + AddFilterChainDataForDestinationIpRange( + filter_chain, &internal_filter_chain_map.destination_ip_map, errors); } - *filter_chain_map = - BuildFromInternalFilterChainMap(&internal_filter_chain_map); - return GRPC_ERROR_NONE; + return BuildFromInternalFilterChainMap(&internal_filter_chain_map); } -grpc_error_handle LdsResourceParseServer( - const XdsEncodingContext& context, - const envoy_config_listener_v3_Listener* listener, bool is_v2, - XdsListenerResource* lds_update) { - lds_update->type = XdsListenerResource::ListenerType::kTcpListener; - grpc_error_handle error = - AddressParse(envoy_config_listener_v3_Listener_address(listener), - &lds_update->address); - if (error != GRPC_ERROR_NONE) return error; - const auto* use_original_dst = - envoy_config_listener_v3_Listener_use_original_dst(listener); - if (use_original_dst != nullptr) { - if (google_protobuf_BoolValue_value(use_original_dst)) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Field \'use_original_dst\' is not supported."); +absl::StatusOr<XdsListenerResource> LdsResourceParseServer( + const XdsResourceType::DecodeContext& context, + const envoy_config_listener_v3_Listener* listener) { + ValidationErrors errors; + XdsListenerResource::TcpListener tcp_listener; + // address + { + ValidationErrors::ScopedField field(&errors, "address"); + auto address = AddressParse( + envoy_config_listener_v3_Listener_address(listener), &errors); + if (address.has_value()) tcp_listener.address = std::move(*address); + } + // use_original_dst + { + ValidationErrors::ScopedField field(&errors, "use_original_dst"); + const auto* use_original_dst = + envoy_config_listener_v3_Listener_use_original_dst(listener); + if (use_original_dst != nullptr && + google_protobuf_BoolValue_value(use_original_dst)) { + errors.AddError("field not supported"); } } - size_t size = 0; - auto* filter_chains = - envoy_config_listener_v3_Listener_filter_chains(listener, &size); - std::vector<FilterChain> parsed_filter_chains; - parsed_filter_chains.reserve(size); - for (size_t i = 0; i < size; i++) { - FilterChain filter_chain; - error = FilterChainParse(context, filter_chains[i], is_v2, &filter_chain); - if (error != GRPC_ERROR_NONE) return error; - parsed_filter_chains.push_back(std::move(filter_chain)); - } - error = - BuildFilterChainMap(parsed_filter_chains, &lds_update->filter_chain_map); - if (error != GRPC_ERROR_NONE) return error; - auto* default_filter_chain = - envoy_config_listener_v3_Listener_default_filter_chain(listener); - if (default_filter_chain != nullptr) { - FilterChain filter_chain; - error = - FilterChainParse(context, default_filter_chain, is_v2, &filter_chain); - if (error != GRPC_ERROR_NONE) return error; - if (filter_chain.filter_chain_data != nullptr) { - lds_update->default_filter_chain = - std::move(*filter_chain.filter_chain_data); + // filter_chains + size_t num_filter_chains = 0; + { + ValidationErrors::ScopedField field(&errors, "filter_chains"); + auto* filter_chains = envoy_config_listener_v3_Listener_filter_chains( + listener, &num_filter_chains); + std::vector<FilterChain> parsed_filter_chains; + parsed_filter_chains.reserve(num_filter_chains); + for (size_t i = 0; i < num_filter_chains; i++) { + ValidationErrors::ScopedField field(&errors, absl::StrCat("[", i, "]")); + auto filter_chain = FilterChainParse(context, filter_chains[i], &errors); + if (filter_chain.has_value()) { + parsed_filter_chains.push_back(std::move(*filter_chain)); + } } + tcp_listener.filter_chain_map = + BuildFilterChainMap(parsed_filter_chains, &errors); } - if (size == 0 && default_filter_chain == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("No filter chain provided."); + // default_filter_chain + { + ValidationErrors::ScopedField field(&errors, "default_filter_chain"); + auto* default_filter_chain = + envoy_config_listener_v3_Listener_default_filter_chain(listener); + if (default_filter_chain != nullptr) { + auto filter_chain = + FilterChainParse(context, default_filter_chain, &errors); + if (filter_chain.has_value() && + filter_chain->filter_chain_data != nullptr) { + tcp_listener.default_filter_chain = + std::move(*filter_chain->filter_chain_data); + } + } else if (num_filter_chains == 0) { + // Make sure that there is at least one filter chain to use. + errors.AddError("must be set if filter_chains is unset"); + } } - return GRPC_ERROR_NONE; + // Return result. + if (!errors.ok()) { + return errors.status(absl::StatusCode::kInvalidArgument, + "errors validating server Listener"); + } + XdsListenerResource lds_update; + lds_update.listener = std::move(tcp_listener); + return lds_update; } -grpc_error_handle LdsResourceParse( - const XdsEncodingContext& context, - const envoy_config_listener_v3_Listener* listener, bool is_v2, - XdsListenerResource* lds_update) { +absl::StatusOr<XdsListenerResource> LdsResourceParse( + const XdsResourceType::DecodeContext& context, + const envoy_config_listener_v3_Listener* listener) { // Check whether it's a client or server listener. const envoy_config_listener_v3_ApiListener* api_listener = envoy_config_listener_v3_Listener_api_listener(listener); const envoy_config_core_v3_Address* address = envoy_config_listener_v3_Listener_address(listener); - if (api_listener != nullptr && address != nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Listener has both address and ApiListener"); - } + // TODO(roth): Re-enable the following check once + // github.com/istio/istio/issues/38914 is resolved. + // if (api_listener != nullptr && address != nullptr) { + // return absl::InvalidArgumentError( + // "Listener has both address and ApiListener"); + // } if (api_listener == nullptr && address == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( + return absl::InvalidArgumentError( "Listener has neither address nor ApiListener"); } - // Validate Listener fields. - grpc_error_handle error = GRPC_ERROR_NONE; + // If api_listener is present, it's for a client; otherwise, it's + // for a server. if (api_listener != nullptr) { - error = LdsResourceParseClient(context, api_listener, is_v2, lds_update); - } else { - error = LdsResourceParseServer(context, listener, is_v2, lds_update); + return LdsResourceParseClient(context, api_listener); } - return error; + return LdsResourceParseServer(context, listener); } -void MaybeLogListener(const XdsEncodingContext& context, +void MaybeLogListener(const XdsResourceType::DecodeContext& context, const envoy_config_listener_v3_Listener* listener) { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { @@ -1000,40 +1099,40 @@ void MaybeLogListener(const XdsEncodingContext& context, } // namespace -absl::StatusOr<XdsResourceType::DecodeResult> XdsListenerResourceType::Decode( - const XdsEncodingContext& context, absl::string_view serialized_resource, - bool is_v2) const { +XdsResourceType::DecodeResult XdsListenerResourceType::Decode( + const XdsResourceType::DecodeContext& context, + absl::string_view serialized_resource) const { + DecodeResult result; // Parse serialized proto. auto* resource = envoy_config_listener_v3_Listener_parse( serialized_resource.data(), serialized_resource.size(), context.arena); if (resource == nullptr) { - return absl::InvalidArgumentError("Can't parse Listener resource."); + result.resource = + absl::InvalidArgumentError("Can't parse Listener resource."); + return result; } MaybeLogListener(context, resource); // Validate resource. - DecodeResult result; result.name = UpbStringToStdString(envoy_config_listener_v3_Listener_name(resource)); - auto listener_data = absl::make_unique<ResourceDataSubclass>(); - grpc_error_handle error = - LdsResourceParse(context, resource, is_v2, &listener_data->resource); - if (error != GRPC_ERROR_NONE) { - std::string error_str = grpc_error_std_string(error); - GRPC_ERROR_UNREF(error); + auto listener = LdsResourceParse(context, resource); + if (!listener.ok()) { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer)) { gpr_log(GPR_ERROR, "[xds_client %p] invalid Listener %s: %s", - context.client, result.name.c_str(), error_str.c_str()); + context.client, result.name->c_str(), + listener.status().ToString().c_str()); } - result.resource = absl::InvalidArgumentError(error_str); + result.resource = listener.status(); } else { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer)) { gpr_log(GPR_INFO, "[xds_client %p] parsed Listener %s: %s", - context.client, result.name.c_str(), - listener_data->resource.ToString().c_str()); + context.client, result.name->c_str(), + listener->ToString().c_str()); } - result.resource = std::move(listener_data); + result.resource = + std::make_unique<XdsListenerResource>(std::move(*listener)); } - return std::move(result); + return result; } } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_listener.h b/grpc/src/core/ext/xds/xds_listener.h index dd9bb450..bc5e8d86 100644 --- a/grpc/src/core/ext/xds/xds_listener.h +++ b/grpc/src/core/ext/xds/xds_listener.h @@ -14,60 +14,48 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_LISTENER_H -#define GRPC_CORE_EXT_XDS_XDS_LISTENER_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_LISTENER_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_LISTENER_H #include <grpc/support/port_platform.h> +#include <stdint.h> +#include <string.h> + +#include <algorithm> #include <array> #include <map> +#include <memory> #include <string> #include <vector> -#include "absl/status/statusor.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" +#include "absl/types/variant.h" #include "envoy/config/listener/v3/listener.upbdefs.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.upbdefs.h" +#include "upb/reflection/def.h" +#include "src/core/ext/xds/xds_bootstrap_grpc.h" #include "src/core/ext/xds/xds_client.h" #include "src/core/ext/xds/xds_common_types.h" #include "src/core/ext/xds/xds_http_filters.h" +#include "src/core/ext/xds/xds_resource_type.h" #include "src/core/ext/xds/xds_resource_type_impl.h" #include "src/core/ext/xds/xds_route_config.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/iomgr/resolved_address.h" namespace grpc_core { -// TODO(roth): When we can use absl::variant<>, consider using that -// here, to enforce the fact that only one of the two fields can be set. -struct XdsListenerResource { - struct DownstreamTlsContext { - CommonTlsContext common_tls_context; - bool require_client_certificate = false; - - bool operator==(const DownstreamTlsContext& other) const { - return common_tls_context == other.common_tls_context && - require_client_certificate == other.require_client_certificate; - } - - std::string ToString() const; - bool Empty() const; - }; - - enum class ListenerType { - kTcpListener = 0, - kHttpApiListener, - } type; - +struct XdsListenerResource : public XdsResourceType::ResourceData { struct HttpConnectionManager { - // The name to use in the RDS request. - std::string route_config_name; + // The RDS resource name or inline RouteConfiguration. + absl::variant<std::string, XdsRouteConfigResource> route_config; + // Storing the Http Connection Manager Common Http Protocol Option // max_stream_duration Duration http_max_stream_duration; - // The RouteConfiguration to use for this listener. - // Present only if it is inlined in the LDS response. - absl::optional<XdsRouteConfigResource> rds_update; struct HttpFilter { std::string name; @@ -82,21 +70,28 @@ struct XdsListenerResource { std::vector<HttpFilter> http_filters; bool operator==(const HttpConnectionManager& other) const { - return route_config_name == other.route_config_name && + return route_config == other.route_config && http_max_stream_duration == other.http_max_stream_duration && - rds_update == other.rds_update && http_filters == other.http_filters; } std::string ToString() const; }; - // Populated for type=kHttpApiListener. - HttpConnectionManager http_connection_manager; + struct DownstreamTlsContext { + DownstreamTlsContext() {} + + CommonTlsContext common_tls_context; + bool require_client_certificate = false; - // Populated for type=kTcpListener. - // host:port listening_address set when type is kTcpListener - std::string address; + bool operator==(const DownstreamTlsContext& other) const { + return common_tls_context == other.common_tls_context && + require_client_certificate == other.require_client_certificate; + } + + std::string ToString() const; + bool Empty() const; + }; struct FilterChainData { DownstreamTlsContext downstream_tls_context; @@ -177,15 +172,26 @@ struct XdsListenerResource { } std::string ToString() const; - } filter_chain_map; + }; - absl::optional<FilterChainData> default_filter_chain; + struct TcpListener { + std::string address; // host:port listening address + FilterChainMap filter_chain_map; + absl::optional<FilterChainData> default_filter_chain; + + bool operator==(const TcpListener& other) const { + return address == other.address && + filter_chain_map == other.filter_chain_map && + default_filter_chain == other.default_filter_chain; + } + + std::string ToString() const; + }; + + absl::variant<HttpConnectionManager, TcpListener> listener; bool operator==(const XdsListenerResource& other) const { - return http_connection_manager == other.http_connection_manager && - address == other.address && - filter_chain_map == other.filter_chain_map && - default_filter_chain == other.default_filter_chain; + return listener == other.listener; } std::string ToString() const; @@ -197,24 +203,24 @@ class XdsListenerResourceType absl::string_view type_url() const override { return "envoy.config.listener.v3.Listener"; } - absl::string_view v2_type_url() const override { - return "envoy.api.v2.Listener"; - } - absl::StatusOr<DecodeResult> Decode(const XdsEncodingContext& context, - absl::string_view serialized_resource, - bool is_v2) const override; + DecodeResult Decode(const XdsResourceType::DecodeContext& context, + absl::string_view serialized_resource) const override; bool AllResourcesRequiredInSotW() const override { return true; } - void InitUpbSymtab(upb_DefPool* symtab) const override { + void InitUpbSymtab(XdsClient* xds_client, + upb_DefPool* symtab) const override { envoy_config_listener_v3_Listener_getmsgdef(symtab); envoy_extensions_filters_network_http_connection_manager_v3_HttpConnectionManager_getmsgdef( symtab); - XdsHttpFilterRegistry::PopulateSymtab(symtab); + const auto& http_filter_registry = + static_cast<const GrpcXdsBootstrap&>(xds_client->bootstrap()) + .http_filter_registry(); + http_filter_registry.PopulateSymtab(symtab); } }; } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_LISTENER_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_LISTENER_H diff --git a/grpc/src/core/ext/xds/xds_resource_type.cc b/grpc/src/core/ext/xds/xds_resource_type.cc deleted file mode 100644 index 5dbb36b3..00000000 --- a/grpc/src/core/ext/xds/xds_resource_type.cc +++ /dev/null @@ -1,33 +0,0 @@ -// -// Copyright 2021 gRPC authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -#include <grpc/support/port_platform.h> - -#include "src/core/ext/xds/xds_resource_type.h" - -namespace grpc_core { - -bool XdsResourceType::IsType(absl::string_view resource_type, - bool* is_v2) const { - if (resource_type == type_url()) return true; - if (resource_type == v2_type_url()) { - if (is_v2 != nullptr) *is_v2 = true; - return true; - } - return false; -} - -} // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_resource_type.h b/grpc/src/core/ext/xds/xds_resource_type.h index ddaf56d4..4a9f96ad 100644 --- a/grpc/src/core/ext/xds/xds_resource_type.h +++ b/grpc/src/core/ext/xds/xds_resource_type.h @@ -14,6 +14,8 @@ // limitations under the License. // +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_H #include <grpc/support/port_platform.h> #include <memory> @@ -21,18 +23,30 @@ #include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "upb/mem/arena.h" +#include "upb/reflection/def.h" -#include "src/core/ext/xds/upb_utils.h" - -#ifndef GRPC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_H -#define GRPC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_H +#include "src/core/ext/xds/xds_bootstrap.h" +#include "src/core/lib/debug/trace.h" namespace grpc_core { +class XdsClient; + // Interface for an xDS resource type. // Used to inject type-specific logic into XdsClient. class XdsResourceType { public: + // Context passed into Decode(). + struct DecodeContext { + XdsClient* client; + const XdsBootstrap::XdsServer& server; + TraceFlag* tracer; + upb_DefPool* symtab; + upb_Arena* arena; + }; + // A base type for resource data. // Subclasses will extend this, and their DecodeResults will be // downcastable to their extended type. @@ -42,7 +56,11 @@ class XdsResourceType { // Result returned by Decode(). struct DecodeResult { - std::string name; + // The resource's name, if it can be determined. + // If the name is not returned, the resource field should contain a + // non-OK status. + absl::optional<std::string> name; + // The parsed and validated resource, or an error status. absl::StatusOr<std::unique_ptr<ResourceData>> resource; }; @@ -51,17 +69,9 @@ class XdsResourceType { // Returns v3 resource type. virtual absl::string_view type_url() const = 0; - // Returns v2 resource type. - virtual absl::string_view v2_type_url() const = 0; - // Decodes and validates a serialized resource proto. - // If the resource fails protobuf deserialization, returns non-OK status. - // If the deserialized resource fails validation, returns a DecodeResult - // whose resource field is set to a non-OK status. - // Otherwise, returns a DecodeResult with a valid resource. - virtual absl::StatusOr<DecodeResult> Decode( - const XdsEncodingContext& context, absl::string_view serialized_resource, - bool is_v2) const = 0; + virtual DecodeResult Decode(const DecodeContext& context, + absl::string_view serialized_resource) const = 0; // Returns true if r1 and r2 are equal. // Must be invoked only on resources returned by this object's Decode() @@ -85,14 +95,10 @@ class XdsResourceType { // properly in logs. // Note: This won't actually work properly until upb adds support for // Any fields in textproto printing (internal b/178821188). - virtual void InitUpbSymtab(upb_DefPool* symtab) const = 0; - - // Convenience method for checking if resource_type matches this type. - // Checks against both type_url() and v2_type_url(). - // If is_v2 is non-null, it will be set to true if matching v2_type_url(). - bool IsType(absl::string_view resource_type, bool* is_v2) const; + virtual void InitUpbSymtab(XdsClient* xds_client, + upb_DefPool* symtab) const = 0; }; } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_H diff --git a/grpc/src/core/ext/xds/xds_resource_type_impl.h b/grpc/src/core/ext/xds/xds_resource_type_impl.h index 94ebe8ed..5c34ee05 100644 --- a/grpc/src/core/ext/xds/xds_resource_type_impl.h +++ b/grpc/src/core/ext/xds/xds_resource_type_impl.h @@ -14,37 +14,40 @@ // limitations under the License. // +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_IMPL_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_IMPL_H #include <grpc/support/port_platform.h> +#include <memory> + +#include "absl/strings/string_view.h" + #include "src/core/ext/xds/xds_client.h" #include "src/core/ext/xds/xds_resource_type.h" - -#ifndef GRPC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_IMPL_H -#define GRPC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_IMPL_H +#include "src/core/lib/gprpp/ref_counted_ptr.h" namespace grpc_core { // Base class for XdsResourceType implementations. // Handles all down-casting logic for a particular resource type struct. +// ResourceTypeStruct must inherit from XdsResourceType::ResourceData, +// must be copy-constructible, and must implement operator==(). template <typename Subclass, typename ResourceTypeStruct> class XdsResourceTypeImpl : public XdsResourceType { public: - struct ResourceDataSubclass : public ResourceData { - ResourceTypeStruct resource; - }; + using ResourceType = ResourceTypeStruct; // XdsClient watcher that handles down-casting. class WatcherInterface : public XdsClient::ResourceWatcherInterface { public: - virtual void OnResourceChanged(ResourceTypeStruct listener) = 0; + virtual void OnResourceChanged(ResourceType listener) = 0; private: // Get result from XdsClient generic watcher interface, perform - // down-casting, and invoke the caller's OnListenerChanged() method. + // down-casting, and invoke the caller's OnResourceChanged() method. void OnGenericResourceChanged( const XdsResourceType::ResourceData* resource) override { - OnResourceChanged( - static_cast<const ResourceDataSubclass*>(resource)->resource); + OnResourceChanged(*static_cast<const ResourceType*>(resource)); } }; @@ -69,19 +72,17 @@ class XdsResourceTypeImpl : public XdsResourceType { bool ResourcesEqual(const ResourceData* r1, const ResourceData* r2) const override { - return static_cast<const ResourceDataSubclass*>(r1)->resource == - static_cast<const ResourceDataSubclass*>(r2)->resource; + return *static_cast<const ResourceType*>(r1) == + *static_cast<const ResourceType*>(r2); } std::unique_ptr<ResourceData> CopyResource( const ResourceData* resource) const override { - auto* resource_copy = new ResourceDataSubclass(); - resource_copy->resource = - static_cast<const ResourceDataSubclass*>(resource)->resource; - return std::unique_ptr<ResourceData>(resource_copy); + return std::make_unique<ResourceType>( + *static_cast<const ResourceType*>(resource)); } }; } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_IMPL_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_RESOURCE_TYPE_IMPL_H diff --git a/grpc/src/core/ext/xds/xds_route_config.cc b/grpc/src/core/ext/xds/xds_route_config.cc index 0e572c77..f2dc8bad 100644 --- a/grpc/src/core/ext/xds/xds_route_config.cc +++ b/grpc/src/core/ext/xds/xds_route_config.cc @@ -16,56 +16,76 @@ #include <grpc/support/port_platform.h> -#include "absl/memory/memory.h" +#include "src/core/ext/xds/xds_route_config.h" + +#include <stddef.h> +#include <stdint.h> + +#include <initializer_list> +#include <limits> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "absl/status/status.h" +#include "absl/status/statusor.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/str_join.h" #include "absl/strings/str_split.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/variant.h" #include "envoy/config/core/v3/base.upb.h" #include "envoy/config/core/v3/extension.upb.h" #include "envoy/config/route/v3/route.upb.h" #include "envoy/config/route/v3/route.upbdefs.h" #include "envoy/config/route/v3/route_components.upb.h" -#include "envoy/config/route/v3/route_components.upbdefs.h" #include "envoy/type/matcher/v3/regex.upb.h" #include "envoy/type/matcher/v3/string.upb.h" #include "envoy/type/v3/percent.upb.h" #include "envoy/type/v3/range.upb.h" #include "google/protobuf/any.upb.h" +#include "google/protobuf/duration.upb.h" #include "google/protobuf/wrappers.upb.h" -#include "upb/text_encode.h" -#include "upb/upb.h" -#include "upb/upb.hpp" +#include "re2/re2.h" +#include "upb/base/string_view.h" +#include "upb/collections/map.h" +#include "upb/text/encode.h" + +#include <grpc/status.h> +#include <grpc/support/log.h> #include "src/core/ext/xds/upb_utils.h" -#include "src/core/ext/xds/xds_api.h" #include "src/core/ext/xds/xds_cluster_specifier_plugin.h" #include "src/core/ext/xds/xds_common_types.h" +#include "src/core/ext/xds/xds_http_filters.h" #include "src/core/ext/xds/xds_resource_type.h" #include "src/core/ext/xds/xds_routing.h" -#include "src/core/lib/gpr/env.h" +#include "src/core/lib/channel/status_util.h" +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/debug/trace.h" #include "src/core/lib/gpr/string.h" -#include "src/core/lib/iomgr/error.h" -#include "src/core/lib/transport/error_utils.h" +#include "src/core/lib/gprpp/env.h" +#include "src/core/lib/gprpp/match.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/json/json_writer.h" +#include "src/core/lib/load_balancing/lb_policy_registry.h" +#include "src/core/lib/matchers/matchers.h" namespace grpc_core { -// TODO(yashykt): Remove once RBAC is no longer experimental -bool XdsRbacEnabled() { - char* value = gpr_getenv("GRPC_XDS_EXPERIMENTAL_RBAC"); - bool parsed_value; - bool parse_succeeded = gpr_parse_bool_value(value, &parsed_value); - gpr_free(value); - return parse_succeeded && parsed_value; -} - -// TODO(donnadionne): Remove once RLS is no longer experimental +// TODO(apolcyn): remove this flag by the 1.58 release bool XdsRlsEnabled() { - char* value = gpr_getenv("GRPC_EXPERIMENTAL_XDS_RLS_LB"); + auto value = GetEnv("GRPC_EXPERIMENTAL_XDS_RLS_LB"); + if (!value.has_value()) return true; bool parsed_value; - bool parse_succeeded = gpr_parse_bool_value(value, &parsed_value); - gpr_free(value); + bool parse_succeeded = gpr_parse_bool_value(value->c_str(), &parsed_value); return parse_succeeded && parsed_value; } @@ -109,85 +129,77 @@ std::string XdsRouteConfigResource::Route::Matchers::ToString() const { } // -// XdsRouteConfigResource::Route::RouteAction::HashPolicy +// XdsRouteConfigResource::Route::RouteAction::HashPolicy::Header // -XdsRouteConfigResource::Route::RouteAction::HashPolicy::HashPolicy( - const HashPolicy& other) - : type(other.type), - header_name(other.header_name), +XdsRouteConfigResource::Route::RouteAction::HashPolicy::Header::Header( + const Header& other) + : header_name(other.header_name), regex_substitution(other.regex_substitution) { if (other.regex != nullptr) { regex = - absl::make_unique<RE2>(other.regex->pattern(), other.regex->options()); + std::make_unique<RE2>(other.regex->pattern(), other.regex->options()); } } -XdsRouteConfigResource::Route::RouteAction::HashPolicy& -XdsRouteConfigResource::Route::RouteAction::HashPolicy::operator=( - const HashPolicy& other) { - type = other.type; +XdsRouteConfigResource::Route::RouteAction::HashPolicy::Header& +XdsRouteConfigResource::Route::RouteAction::HashPolicy::Header::operator=( + const Header& other) { header_name = other.header_name; if (other.regex != nullptr) { regex = - absl::make_unique<RE2>(other.regex->pattern(), other.regex->options()); + std::make_unique<RE2>(other.regex->pattern(), other.regex->options()); } regex_substitution = other.regex_substitution; return *this; } -XdsRouteConfigResource::Route::RouteAction::HashPolicy::HashPolicy( - HashPolicy&& other) noexcept - : type(other.type), - header_name(std::move(other.header_name)), +XdsRouteConfigResource::Route::RouteAction::HashPolicy::Header::Header( + Header&& other) noexcept + : header_name(std::move(other.header_name)), regex(std::move(other.regex)), regex_substitution(std::move(other.regex_substitution)) {} -XdsRouteConfigResource::Route::RouteAction::HashPolicy& -XdsRouteConfigResource::Route::RouteAction::HashPolicy::operator=( - HashPolicy&& other) noexcept { - type = other.type; +XdsRouteConfigResource::Route::RouteAction::HashPolicy::Header& +XdsRouteConfigResource::Route::RouteAction::HashPolicy::Header::operator=( + Header&& other) noexcept { header_name = std::move(other.header_name); regex = std::move(other.regex); regex_substitution = std::move(other.regex_substitution); return *this; } -bool XdsRouteConfigResource::Route::RouteAction::HashPolicy::HashPolicy:: -operator==(const HashPolicy& other) const { - if (type != other.type) return false; - if (type == Type::HEADER) { - if (regex == nullptr) { - if (other.regex != nullptr) return false; - } else { - if (other.regex == nullptr) return false; - return header_name == other.header_name && - regex->pattern() == other.regex->pattern() && - regex_substitution == other.regex_substitution; - } +bool XdsRouteConfigResource::Route::RouteAction::HashPolicy::Header::operator==( + const Header& other) const { + if (header_name != other.header_name) return false; + if (regex == nullptr) { + if (other.regex != nullptr) return false; + } else { + if (other.regex == nullptr) return false; + if (regex->pattern() != other.regex->pattern()) return false; } - return true; + return regex_substitution == other.regex_substitution; } +std::string +XdsRouteConfigResource::Route::RouteAction::HashPolicy::Header::ToString() + const { + return absl::StrCat("Header ", header_name, "/", + (regex == nullptr) ? "" : regex->pattern(), "/", + regex_substitution); +} + +// +// XdsRouteConfigResource::Route::RouteAction::HashPolicy +// + std::string XdsRouteConfigResource::Route::RouteAction::HashPolicy::ToString() const { - std::vector<std::string> contents; - switch (type) { - case Type::HEADER: - contents.push_back("type=HEADER"); - break; - case Type::CHANNEL_ID: - contents.push_back("type=CHANNEL_ID"); - break; - } - contents.push_back( - absl::StrFormat("terminal=%s", terminal ? "true" : "false")); - if (type == Type::HEADER) { - contents.push_back(absl::StrFormat( - "Header %s:/%s/%s", header_name, - (regex == nullptr) ? "" : regex->pattern(), regex_substitution)); - } - return absl::StrCat("{", absl::StrJoin(contents, ", "), "}"); + std::string type = Match( + policy, [](const Header& header) { return header.ToString(); }, + [](const ChannelId&) -> std::string { return "ChannelId"; }); + return absl::StrCat("{", type, ", terminal=", terminal ? "true" : "false", + "}"); } // @@ -218,25 +230,30 @@ XdsRouteConfigResource::Route::RouteAction::ClusterWeight::ToString() const { std::string XdsRouteConfigResource::Route::RouteAction::ToString() const { std::vector<std::string> contents; + contents.reserve(hash_policies.size()); for (const HashPolicy& hash_policy : hash_policies) { contents.push_back(absl::StrCat("hash_policy=", hash_policy.ToString())); } if (retry_policy.has_value()) { contents.push_back(absl::StrCat("retry_policy=", retry_policy->ToString())); } - if (action.index() == kClusterIndex) { - contents.push_back( - absl::StrFormat("Cluster name: %s", absl::get<kClusterIndex>(action))); - } else if (action.index() == kWeightedClustersIndex) { - auto& action_weighted_clusters = absl::get<kWeightedClustersIndex>(action); - for (const ClusterWeight& cluster_weight : action_weighted_clusters) { - contents.push_back(cluster_weight.ToString()); - } - } else if (action.index() == kClusterSpecifierPluginIndex) { - contents.push_back( - absl::StrFormat("Cluster specifier plugin name: %s", - absl::get<kClusterSpecifierPluginIndex>(action))); - } + Match( + action, + [&contents](const ClusterName& cluster_name) { + contents.push_back( + absl::StrFormat("Cluster name: %s", cluster_name.cluster_name)); + }, + [&contents](const std::vector<ClusterWeight>& weighted_clusters) { + for (const ClusterWeight& cluster_weight : weighted_clusters) { + contents.push_back(cluster_weight.ToString()); + } + }, + [&contents]( + const ClusterSpecifierPluginName& cluster_specifier_plugin_name) { + contents.push_back(absl::StrFormat( + "Cluster specifier plugin name: %s", + cluster_specifier_plugin_name.cluster_specifier_plugin_name)); + }); if (max_stream_duration.has_value()) { contents.push_back(max_stream_duration->ToString()); } @@ -310,68 +327,85 @@ std::string XdsRouteConfigResource::ToString() const { namespace { -grpc_error_handle ClusterSpecifierPluginParse( - const XdsEncodingContext& context, +XdsRouteConfigResource::ClusterSpecifierPluginMap ClusterSpecifierPluginParse( + const XdsResourceType::DecodeContext& context, const envoy_config_route_v3_RouteConfiguration* route_config, - XdsRouteConfigResource* rds_update) { + ValidationErrors* errors) { + XdsRouteConfigResource::ClusterSpecifierPluginMap + cluster_specifier_plugin_map; + const auto& cluster_specifier_plugin_registry = + static_cast<const GrpcXdsBootstrap&>(context.client->bootstrap()) + .cluster_specifier_plugin_registry(); size_t num_cluster_specifier_plugins; const envoy_config_route_v3_ClusterSpecifierPlugin* const* cluster_specifier_plugin = envoy_config_route_v3_RouteConfiguration_cluster_specifier_plugins( route_config, &num_cluster_specifier_plugins); for (size_t i = 0; i < num_cluster_specifier_plugins; ++i) { - const envoy_config_core_v3_TypedExtensionConfig* extension = + bool is_optional = envoy_config_route_v3_ClusterSpecifierPlugin_is_optional( + cluster_specifier_plugin[i]); + ValidationErrors::ScopedField field( + errors, absl::StrCat(".cluster_specifier_plugins[", i, "].extension")); + const envoy_config_core_v3_TypedExtensionConfig* typed_extension_config = envoy_config_route_v3_ClusterSpecifierPlugin_extension( cluster_specifier_plugin[i]); std::string name = UpbStringToStdString( - envoy_config_core_v3_TypedExtensionConfig_name(extension)); - if (rds_update->cluster_specifier_plugin_map.find(name) != - rds_update->cluster_specifier_plugin_map.end()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat( - "Duplicated definition of cluster_specifier_plugin ", name)); + envoy_config_core_v3_TypedExtensionConfig_name(typed_extension_config)); + if (cluster_specifier_plugin_map.find(name) != + cluster_specifier_plugin_map.end()) { + ValidationErrors::ScopedField field(errors, ".name"); + errors->AddError(absl::StrCat("duplicate name \"", name, "\"")); + } else { + // Add a sentinel entry in case we encounter an error later, just so we + // don't generate duplicate errors for each route that uses this plugin. + cluster_specifier_plugin_map[name] = "<sentinel>"; } + ValidationErrors::ScopedField field2(errors, ".typed_config"); const google_protobuf_Any* any = - envoy_config_core_v3_TypedExtensionConfig_typed_config(extension); - if (any == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Could not obtrain TypedExtensionConfig for plugin config."); - } - absl::string_view plugin_type; - grpc_error_handle error = - ExtractExtensionTypeName(context, any, &plugin_type); - if (error != GRPC_ERROR_NONE) return error; - bool is_optional = envoy_config_route_v3_ClusterSpecifierPlugin_is_optional( - cluster_specifier_plugin[i]); + envoy_config_core_v3_TypedExtensionConfig_typed_config( + typed_extension_config); + auto extension = ExtractXdsExtension(context, any, errors); + if (!extension.has_value()) continue; const XdsClusterSpecifierPluginImpl* cluster_specifier_plugin_impl = - XdsClusterSpecifierPluginRegistry::GetPluginForType(plugin_type); - std::string lb_policy_config; + cluster_specifier_plugin_registry.GetPluginForType(extension->type); if (cluster_specifier_plugin_impl == nullptr) { - if (!is_optional) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("Unknown ClusterSpecifierPlugin type ", plugin_type)); + if (is_optional) { + // Empty string indicates an optional plugin. + // This is used later when validating routes, and since we will skip + // any routes that refer to this plugin, we won't wind up including + // this plugin in the resource that we return to the watcher. + cluster_specifier_plugin_map[std::move(name)] = ""; + } else { + // Not optional, report error. + errors->AddError("unsupported ClusterSpecifierPlugin type"); } - // Optional plugin, leave lb_policy_config empty. + continue; + } + const size_t original_error_size = errors->size(); + Json lb_policy_config = + cluster_specifier_plugin_impl->GenerateLoadBalancingPolicyConfig( + std::move(*extension), context.arena, context.symtab, errors); + if (errors->size() != original_error_size) continue; + auto config = + CoreConfiguration::Get().lb_policy_registry().ParseLoadBalancingConfig( + lb_policy_config); + if (!config.ok()) { + errors->AddError(absl::StrCat( + "ClusterSpecifierPlugin returned invalid LB policy config: ", + config.status().message())); } else { - auto config = - cluster_specifier_plugin_impl->GenerateLoadBalancingPolicyConfig( - google_protobuf_Any_value(any), context.arena, context.symtab); - if (!config.ok()) { - return absl_status_to_grpc_error(config.status()); - } - lb_policy_config = std::move(*config); + cluster_specifier_plugin_map[std::move(name)] = + JsonDump(lb_policy_config); } - rds_update->cluster_specifier_plugin_map[std::move(name)] = - std::move(lb_policy_config); } - return GRPC_ERROR_NONE; + return cluster_specifier_plugin_map; } -grpc_error_handle RoutePathMatchParse( - const envoy_config_route_v3_RouteMatch* match, - XdsRouteConfigResource::Route* route, bool* ignore_route) { +absl::optional<StringMatcher> RoutePathMatchParse( + const envoy_config_route_v3_RouteMatch* match, ValidationErrors* errors) { + bool case_sensitive = true; auto* case_sensitive_ptr = envoy_config_route_v3_RouteMatch_case_sensitive(match); - bool case_sensitive = true; if (case_sensitive_ptr != nullptr) { case_sensitive = google_protobuf_BoolValue_value(case_sensitive_ptr); } @@ -380,25 +414,18 @@ grpc_error_handle RoutePathMatchParse( if (envoy_config_route_v3_RouteMatch_has_prefix(match)) { absl::string_view prefix = UpbStringToAbsl(envoy_config_route_v3_RouteMatch_prefix(match)); - // Empty prefix "" is accepted. + // For any prefix that cannot match a path of the form "/service/method", + // ignore the route. if (!prefix.empty()) { - // Prefix "/" is accepted. - if (prefix[0] != '/') { - // Prefix which does not start with a / will never match anything, so - // ignore this route. - *ignore_route = true; - return GRPC_ERROR_NONE; - } + // Does not start with a slash. + if (prefix[0] != '/') return absl::nullopt; std::vector<absl::string_view> prefix_elements = absl::StrSplit(prefix.substr(1), absl::MaxSplits('/', 2)); - if (prefix_elements.size() > 2) { - // Prefix cannot have more than 2 slashes. - *ignore_route = true; - return GRPC_ERROR_NONE; - } else if (prefix_elements.size() == 2 && prefix_elements[0].empty()) { - // Prefix contains empty string between the 2 slashes - *ignore_route = true; - return GRPC_ERROR_NONE; + // More than 2 slashes. + if (prefix_elements.size() > 2) return absl::nullopt; + // Two consecutive slashes. + if (prefix_elements.size() == 2 && prefix_elements[0].empty()) { + return absl::nullopt; } } type = StringMatcher::Type::kPrefix; @@ -406,35 +433,19 @@ grpc_error_handle RoutePathMatchParse( } else if (envoy_config_route_v3_RouteMatch_has_path(match)) { absl::string_view path = UpbStringToAbsl(envoy_config_route_v3_RouteMatch_path(match)); - if (path.empty()) { - // Path that is empty will never match anything, so ignore this route. - *ignore_route = true; - return GRPC_ERROR_NONE; - } - if (path[0] != '/') { - // Path which does not start with a / will never match anything, so - // ignore this route. - *ignore_route = true; - return GRPC_ERROR_NONE; - } + // For any path not of the form "/service/method", ignore the route. + // Empty path. + if (path.empty()) return absl::nullopt; + // Does not start with a slash. + if (path[0] != '/') return absl::nullopt; std::vector<absl::string_view> path_elements = absl::StrSplit(path.substr(1), absl::MaxSplits('/', 2)); - if (path_elements.size() != 2) { - // Path not in the required format of /service/method will never match - // anything, so ignore this route. - *ignore_route = true; - return GRPC_ERROR_NONE; - } else if (path_elements[0].empty()) { - // Path contains empty service name will never match anything, so ignore - // this route. - *ignore_route = true; - return GRPC_ERROR_NONE; - } else if (path_elements[1].empty()) { - // Path contains empty method name will never match anything, so ignore - // this route. - *ignore_route = true; - return GRPC_ERROR_NONE; - } + // Number of slashes does not equal 2. + if (path_elements.size() != 2) return absl::nullopt; + // Empty service name. + if (path_elements[0].empty()) return absl::nullopt; + // Empty method name. + if (path_elements[1].empty()) return absl::nullopt; type = StringMatcher::Type::kExact; match_string = std::string(path); } else if (envoy_config_route_v3_RouteMatch_has_safe_regex(match)) { @@ -445,27 +456,30 @@ grpc_error_handle RoutePathMatchParse( match_string = UpbStringToStdString( envoy_type_matcher_v3_RegexMatcher_regex(regex_matcher)); } else { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Invalid route path specifier specified."); + errors->AddError("invalid path specifier"); + return absl::nullopt; } absl::StatusOr<StringMatcher> string_matcher = StringMatcher::Create(type, match_string, case_sensitive); if (!string_matcher.ok()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("path matcher: ", string_matcher.status().message())); + errors->AddError(absl::StrCat("error creating path matcher: ", + string_matcher.status().message())); + return absl::nullopt; } - route->matchers.path_matcher = std::move(string_matcher.value()); - return GRPC_ERROR_NONE; + return std::move(*string_matcher); } -grpc_error_handle RouteHeaderMatchersParse( - const envoy_config_route_v3_RouteMatch* match, - XdsRouteConfigResource::Route* route) { +void RouteHeaderMatchersParse(const envoy_config_route_v3_RouteMatch* match, + XdsRouteConfigResource::Route* route, + ValidationErrors* errors) { size_t size; const envoy_config_route_v3_HeaderMatcher* const* headers = envoy_config_route_v3_RouteMatch_headers(match, &size); for (size_t i = 0; i < size; ++i) { + ValidationErrors::ScopedField field(errors, + absl::StrCat(".headers[", i, "]")); const envoy_config_route_v3_HeaderMatcher* header = headers[i]; + GPR_ASSERT(header != nullptr); const std::string name = UpbStringToStdString(envoy_config_route_v3_HeaderMatcher_name(header)); HeaderMatcher::Type type; @@ -473,10 +487,23 @@ grpc_error_handle RouteHeaderMatchersParse( int64_t range_start = 0; int64_t range_end = 0; bool present_match = false; + bool case_sensitive = true; if (envoy_config_route_v3_HeaderMatcher_has_exact_match(header)) { type = HeaderMatcher::Type::kExact; match_string = UpbStringToStdString( envoy_config_route_v3_HeaderMatcher_exact_match(header)); + } else if (envoy_config_route_v3_HeaderMatcher_has_prefix_match(header)) { + type = HeaderMatcher::Type::kPrefix; + match_string = UpbStringToStdString( + envoy_config_route_v3_HeaderMatcher_prefix_match(header)); + } else if (envoy_config_route_v3_HeaderMatcher_has_suffix_match(header)) { + type = HeaderMatcher::Type::kSuffix; + match_string = UpbStringToStdString( + envoy_config_route_v3_HeaderMatcher_suffix_match(header)); + } else if (envoy_config_route_v3_HeaderMatcher_has_contains_match(header)) { + type = HeaderMatcher::Type::kContains; + match_string = UpbStringToStdString( + envoy_config_route_v3_HeaderMatcher_contains_match(header)); } else if (envoy_config_route_v3_HeaderMatcher_has_safe_regex_match( header)) { const envoy_type_matcher_v3_RegexMatcher* regex_matcher = @@ -489,45 +516,67 @@ grpc_error_handle RouteHeaderMatchersParse( type = HeaderMatcher::Type::kRange; const envoy_type_v3_Int64Range* range_matcher = envoy_config_route_v3_HeaderMatcher_range_match(header); + GPR_ASSERT(range_matcher != nullptr); range_start = envoy_type_v3_Int64Range_start(range_matcher); range_end = envoy_type_v3_Int64Range_end(range_matcher); } else if (envoy_config_route_v3_HeaderMatcher_has_present_match(header)) { type = HeaderMatcher::Type::kPresent; present_match = envoy_config_route_v3_HeaderMatcher_present_match(header); - } else if (envoy_config_route_v3_HeaderMatcher_has_prefix_match(header)) { - type = HeaderMatcher::Type::kPrefix; - match_string = UpbStringToStdString( - envoy_config_route_v3_HeaderMatcher_prefix_match(header)); - } else if (envoy_config_route_v3_HeaderMatcher_has_suffix_match(header)) { - type = HeaderMatcher::Type::kSuffix; - match_string = UpbStringToStdString( - envoy_config_route_v3_HeaderMatcher_suffix_match(header)); - } else if (envoy_config_route_v3_HeaderMatcher_has_contains_match(header)) { - type = HeaderMatcher::Type::kContains; - match_string = UpbStringToStdString( - envoy_config_route_v3_HeaderMatcher_contains_match(header)); + } else if (envoy_config_route_v3_HeaderMatcher_has_string_match(header)) { + ValidationErrors::ScopedField field(errors, ".string_match"); + const auto* matcher = + envoy_config_route_v3_HeaderMatcher_string_match(header); + GPR_ASSERT(matcher != nullptr); + if (envoy_type_matcher_v3_StringMatcher_has_exact(matcher)) { + type = HeaderMatcher::Type::kExact; + match_string = UpbStringToStdString( + envoy_type_matcher_v3_StringMatcher_exact(matcher)); + } else if (envoy_type_matcher_v3_StringMatcher_has_prefix(matcher)) { + type = HeaderMatcher::Type::kPrefix; + match_string = UpbStringToStdString( + envoy_type_matcher_v3_StringMatcher_prefix(matcher)); + } else if (envoy_type_matcher_v3_StringMatcher_has_suffix(matcher)) { + type = HeaderMatcher::Type::kSuffix; + match_string = UpbStringToStdString( + envoy_type_matcher_v3_StringMatcher_suffix(matcher)); + } else if (envoy_type_matcher_v3_StringMatcher_has_contains(matcher)) { + type = HeaderMatcher::Type::kContains; + match_string = UpbStringToStdString( + envoy_type_matcher_v3_StringMatcher_contains(matcher)); + } else if (envoy_type_matcher_v3_StringMatcher_has_safe_regex(matcher)) { + type = HeaderMatcher::Type::kSafeRegex; + const auto* regex_matcher = + envoy_type_matcher_v3_StringMatcher_safe_regex(matcher); + GPR_ASSERT(regex_matcher != nullptr); + match_string = UpbStringToStdString( + envoy_type_matcher_v3_RegexMatcher_regex(regex_matcher)); + } else { + errors->AddError("invalid string matcher"); + continue; + } + case_sensitive = + !envoy_type_matcher_v3_StringMatcher_ignore_case(matcher); } else { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Invalid route header matcher specified."); + errors->AddError("invalid header matcher"); + continue; } bool invert_match = envoy_config_route_v3_HeaderMatcher_invert_match(header); absl::StatusOr<HeaderMatcher> header_matcher = HeaderMatcher::Create(name, type, match_string, range_start, range_end, - present_match, invert_match); + present_match, invert_match, case_sensitive); if (!header_matcher.ok()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("header matcher: ", header_matcher.status().message())); + errors->AddError(absl::StrCat("cannot create header matcher: ", + header_matcher.status().message())); + } else { + route->matchers.header_matchers.emplace_back(std::move(*header_matcher)); } - route->matchers.header_matchers.emplace_back( - std::move(header_matcher.value())); } - return GRPC_ERROR_NONE; } -grpc_error_handle RouteRuntimeFractionParse( - const envoy_config_route_v3_RouteMatch* match, - XdsRouteConfigResource::Route* route) { +void RouteRuntimeFractionParse(const envoy_config_route_v3_RouteMatch* match, + XdsRouteConfigResource::Route* route, + ValidationErrors* errors) { const envoy_config_core_v3_RuntimeFractionalPercent* runtime_fraction = envoy_config_route_v3_RouteMatch_runtime_fraction(match); if (runtime_fraction != nullptr) { @@ -536,9 +585,8 @@ grpc_error_handle RouteRuntimeFractionParse( runtime_fraction); if (fraction != nullptr) { uint32_t numerator = envoy_type_v3_FractionalPercent_numerator(fraction); - const auto denominator = - static_cast<envoy_type_v3_FractionalPercent_DenominatorType>( - envoy_type_v3_FractionalPercent_denominator(fraction)); + const uint32_t denominator = + envoy_type_v3_FractionalPercent_denominator(fraction); // Normalize to million. switch (denominator) { case envoy_type_v3_FractionalPercent_HUNDRED: @@ -549,101 +597,98 @@ grpc_error_handle RouteRuntimeFractionParse( break; case envoy_type_v3_FractionalPercent_MILLION: break; - default: - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Unknown denominator type"); + default: { + ValidationErrors::ScopedField field( + errors, ".runtime_fraction.default_value.denominator"); + errors->AddError("unknown denominator type"); + return; + } } route->matchers.fraction_per_million = numerator; } } - return GRPC_ERROR_NONE; } template <typename ParentType, typename EntryType> -grpc_error_handle ParseTypedPerFilterConfig( - const XdsEncodingContext& context, const ParentType* parent, +XdsRouteConfigResource::TypedPerFilterConfig ParseTypedPerFilterConfig( + const XdsResourceType::DecodeContext& context, const ParentType* parent, const EntryType* (*entry_func)(const ParentType*, size_t*), upb_StringView (*key_func)(const EntryType*), const google_protobuf_Any* (*value_func)(const EntryType*), - XdsRouteConfigResource::TypedPerFilterConfig* typed_per_filter_config) { + ValidationErrors* errors) { + XdsRouteConfigResource::TypedPerFilterConfig typed_per_filter_config; size_t filter_it = kUpb_Map_Begin; while (true) { const auto* filter_entry = entry_func(parent, &filter_it); if (filter_entry == nullptr) break; absl::string_view key = UpbStringToAbsl(key_func(filter_entry)); - if (key.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("empty filter name in map"); - } + ValidationErrors::ScopedField field(errors, absl::StrCat("[", key, "]")); + if (key.empty()) errors->AddError("filter name must be non-empty"); const google_protobuf_Any* any = value_func(filter_entry); - GPR_ASSERT(any != nullptr); - absl::string_view filter_type = - UpbStringToAbsl(google_protobuf_Any_type_url(any)); - if (filter_type.empty()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("no filter config specified for filter name ", key)); - } + auto extension = ExtractXdsExtension(context, any, errors); + if (!extension.has_value()) continue; + auto* extension_to_use = &*extension; + absl::optional<XdsExtension> nested_extension; bool is_optional = false; - if (filter_type == - "type.googleapis.com/envoy.config.route.v3.FilterConfig") { - upb_StringView any_value = google_protobuf_Any_value(any); + if (extension->type == "envoy.config.route.v3.FilterConfig") { + absl::string_view* serialized_config = + absl::get_if<absl::string_view>(&extension->value); + if (serialized_config == nullptr) { + errors->AddError("could not parse FilterConfig"); + continue; + } const auto* filter_config = envoy_config_route_v3_FilterConfig_parse( - any_value.data, any_value.size, context.arena); + serialized_config->data(), serialized_config->size(), context.arena); if (filter_config == nullptr) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("could not parse FilterConfig wrapper for ", key)); + errors->AddError("could not parse FilterConfig"); + continue; } is_optional = envoy_config_route_v3_FilterConfig_is_optional(filter_config); any = envoy_config_route_v3_FilterConfig_config(filter_config); - if (any == nullptr) { - if (is_optional) continue; - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("no filter config specified for filter name ", key)); - } + extension->validation_fields.emplace_back(errors, ".config"); + nested_extension = ExtractXdsExtension(context, any, errors); + if (!nested_extension.has_value()) continue; + extension_to_use = &*nested_extension; } - grpc_error_handle error = - ExtractExtensionTypeName(context, any, &filter_type); - if (error != GRPC_ERROR_NONE) return error; + const auto& http_filter_registry = + static_cast<const GrpcXdsBootstrap&>(context.client->bootstrap()) + .http_filter_registry(); const XdsHttpFilterImpl* filter_impl = - XdsHttpFilterRegistry::GetFilterForType(filter_type); + http_filter_registry.GetFilterForType(extension_to_use->type); if (filter_impl == nullptr) { - if (is_optional) continue; - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("no filter registered for config type ", filter_type)); + if (!is_optional) errors->AddError("unsupported filter type"); + continue; } - absl::StatusOr<XdsHttpFilterImpl::FilterConfig> filter_config = + absl::optional<XdsHttpFilterImpl::FilterConfig> filter_config = filter_impl->GenerateFilterConfigOverride( - google_protobuf_Any_value(any), context.arena); - if (!filter_config.ok()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat( - "filter config for type ", filter_type, - " failed to parse: ", StatusToString(filter_config.status()))); + context, std::move(*extension_to_use), errors); + if (filter_config.has_value()) { + typed_per_filter_config[std::string(key)] = std::move(*filter_config); } - (*typed_per_filter_config)[std::string(key)] = std::move(*filter_config); } - return GRPC_ERROR_NONE; + return typed_per_filter_config; } -grpc_error_handle RetryPolicyParse( - const XdsEncodingContext& context, - const envoy_config_route_v3_RetryPolicy* retry_policy, - absl::optional<XdsRouteConfigResource::RetryPolicy>* retry) { - std::vector<grpc_error_handle> errors; - XdsRouteConfigResource::RetryPolicy retry_to_return; +XdsRouteConfigResource::RetryPolicy RetryPolicyParse( + const XdsResourceType::DecodeContext& context, + const envoy_config_route_v3_RetryPolicy* retry_policy_proto, + ValidationErrors* errors) { + XdsRouteConfigResource::RetryPolicy retry_policy; auto retry_on = UpbStringToStdString( - envoy_config_route_v3_RetryPolicy_retry_on(retry_policy)); + envoy_config_route_v3_RetryPolicy_retry_on(retry_policy_proto)); std::vector<absl::string_view> codes = absl::StrSplit(retry_on, ','); for (const auto& code : codes) { if (code == "cancelled") { - retry_to_return.retry_on.Add(GRPC_STATUS_CANCELLED); + retry_policy.retry_on.Add(GRPC_STATUS_CANCELLED); } else if (code == "deadline-exceeded") { - retry_to_return.retry_on.Add(GRPC_STATUS_DEADLINE_EXCEEDED); + retry_policy.retry_on.Add(GRPC_STATUS_DEADLINE_EXCEEDED); } else if (code == "internal") { - retry_to_return.retry_on.Add(GRPC_STATUS_INTERNAL); + retry_policy.retry_on.Add(GRPC_STATUS_INTERNAL); } else if (code == "resource-exhausted") { - retry_to_return.retry_on.Add(GRPC_STATUS_RESOURCE_EXHAUSTED); + retry_policy.retry_on.Add(GRPC_STATUS_RESOURCE_EXHAUSTED); } else if (code == "unavailable") { - retry_to_return.retry_on.Add(GRPC_STATUS_UNAVAILABLE); + retry_policy.retry_on.Add(GRPC_STATUS_UNAVAILABLE); } else { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer)) { gpr_log(GPR_INFO, "Unsupported retry_on policy %s.", @@ -652,185 +697,90 @@ grpc_error_handle RetryPolicyParse( } } const google_protobuf_UInt32Value* num_retries = - envoy_config_route_v3_RetryPolicy_num_retries(retry_policy); + envoy_config_route_v3_RetryPolicy_num_retries(retry_policy_proto); if (num_retries != nullptr) { uint32_t num_retries_value = google_protobuf_UInt32Value_value(num_retries); if (num_retries_value == 0) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "RouteAction RetryPolicy num_retries set to invalid value 0.")); + ValidationErrors::ScopedField field(errors, ".num_retries"); + errors->AddError("must be greater than 0"); } else { - retry_to_return.num_retries = num_retries_value; + retry_policy.num_retries = num_retries_value; } } else { - retry_to_return.num_retries = 1; + retry_policy.num_retries = 1; } const envoy_config_route_v3_RetryPolicy_RetryBackOff* backoff = - envoy_config_route_v3_RetryPolicy_retry_back_off(retry_policy); + envoy_config_route_v3_RetryPolicy_retry_back_off(retry_policy_proto); if (backoff != nullptr) { - const google_protobuf_Duration* base_interval = - envoy_config_route_v3_RetryPolicy_RetryBackOff_base_interval(backoff); - if (base_interval == nullptr) { - errors.push_back(GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "RouteAction RetryPolicy RetryBackoff missing base interval.")); - } else { - retry_to_return.retry_back_off.base_interval = - ParseDuration(base_interval); + ValidationErrors::ScopedField field(errors, ".retry_back_off"); + { + ValidationErrors::ScopedField field(errors, ".base_interval"); + const google_protobuf_Duration* base_interval = + envoy_config_route_v3_RetryPolicy_RetryBackOff_base_interval(backoff); + if (base_interval == nullptr) { + errors->AddError("field not present"); + } else { + retry_policy.retry_back_off.base_interval = + ParseDuration(base_interval, errors); + } } - const google_protobuf_Duration* max_interval = - envoy_config_route_v3_RetryPolicy_RetryBackOff_max_interval(backoff); - Duration max; - if (max_interval != nullptr) { - max = ParseDuration(max_interval); - } else { - // if max interval is not set, it is 10x the base. - max = 10 * retry_to_return.retry_back_off.base_interval; + { + ValidationErrors::ScopedField field(errors, ".max_interval"); + const google_protobuf_Duration* max_interval = + envoy_config_route_v3_RetryPolicy_RetryBackOff_max_interval(backoff); + Duration max; + if (max_interval != nullptr) { + max = ParseDuration(max_interval, errors); + } else { + // if max interval is not set, it is 10x the base. + max = 10 * retry_policy.retry_back_off.base_interval; + } + retry_policy.retry_back_off.max_interval = max; } - retry_to_return.retry_back_off.max_interval = max; } else { - retry_to_return.retry_back_off.base_interval = Duration::Milliseconds(25); - retry_to_return.retry_back_off.max_interval = Duration::Milliseconds(250); - } - if (errors.empty()) { - *retry = retry_to_return; - return GRPC_ERROR_NONE; - } else { - return GRPC_ERROR_CREATE_FROM_VECTOR("errors parsing retry policy", - &errors); + retry_policy.retry_back_off.base_interval = Duration::Milliseconds(25); + retry_policy.retry_back_off.max_interval = Duration::Milliseconds(250); } + return retry_policy; } -grpc_error_handle RouteActionParse( - const XdsEncodingContext& context, - const envoy_config_route_v3_Route* route_msg, +absl::optional<XdsRouteConfigResource::Route::RouteAction> RouteActionParse( + const XdsResourceType::DecodeContext& context, + const envoy_config_route_v3_RouteAction* route_action_proto, const std::map<std::string /*cluster_specifier_plugin_name*/, std::string /*LB policy config*/>& cluster_specifier_plugin_map, - XdsRouteConfigResource::Route::RouteAction* route, bool* ignore_route) { - const envoy_config_route_v3_RouteAction* route_action = - envoy_config_route_v3_Route_route(route_msg); - // Get the cluster or weighted_clusters in the RouteAction. - if (envoy_config_route_v3_RouteAction_has_cluster(route_action)) { - std::string cluster_name = UpbStringToStdString( - envoy_config_route_v3_RouteAction_cluster(route_action)); - if (cluster_name.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "RouteAction cluster contains empty cluster name."); - } - route->action - .emplace<XdsRouteConfigResource::Route::RouteAction::kClusterIndex>( - std::move(cluster_name)); - } else if (envoy_config_route_v3_RouteAction_has_weighted_clusters( - route_action)) { - std::vector<XdsRouteConfigResource::Route::RouteAction::ClusterWeight> - action_weighted_clusters; - const envoy_config_route_v3_WeightedCluster* weighted_cluster = - envoy_config_route_v3_RouteAction_weighted_clusters(route_action); - uint32_t total_weight = 100; - const google_protobuf_UInt32Value* weight = - envoy_config_route_v3_WeightedCluster_total_weight(weighted_cluster); - if (weight != nullptr) { - total_weight = google_protobuf_UInt32Value_value(weight); - } - size_t clusters_size; - const envoy_config_route_v3_WeightedCluster_ClusterWeight* const* clusters = - envoy_config_route_v3_WeightedCluster_clusters(weighted_cluster, - &clusters_size); - uint32_t sum_of_weights = 0; - for (size_t j = 0; j < clusters_size; ++j) { - const envoy_config_route_v3_WeightedCluster_ClusterWeight* - cluster_weight = clusters[j]; - XdsRouteConfigResource::Route::RouteAction::ClusterWeight cluster; - cluster.name = UpbStringToStdString( - envoy_config_route_v3_WeightedCluster_ClusterWeight_name( - cluster_weight)); - if (cluster.name.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "RouteAction weighted_cluster cluster contains empty cluster " - "name."); - } - const google_protobuf_UInt32Value* weight = - envoy_config_route_v3_WeightedCluster_ClusterWeight_weight( - cluster_weight); - if (weight == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "RouteAction weighted_cluster cluster missing weight"); - } - cluster.weight = google_protobuf_UInt32Value_value(weight); - if (cluster.weight == 0) continue; - sum_of_weights += cluster.weight; - if (context.use_v3) { - grpc_error_handle error = ParseTypedPerFilterConfig< - envoy_config_route_v3_WeightedCluster_ClusterWeight, - envoy_config_route_v3_WeightedCluster_ClusterWeight_TypedPerFilterConfigEntry>( - context, cluster_weight, - envoy_config_route_v3_WeightedCluster_ClusterWeight_typed_per_filter_config_next, - envoy_config_route_v3_WeightedCluster_ClusterWeight_TypedPerFilterConfigEntry_key, - envoy_config_route_v3_WeightedCluster_ClusterWeight_TypedPerFilterConfigEntry_value, - &cluster.typed_per_filter_config); - if (error != GRPC_ERROR_NONE) return error; - } - action_weighted_clusters.emplace_back(std::move(cluster)); - } - if (total_weight != sum_of_weights) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "RouteAction weighted_cluster has incorrect total weight"); - } - if (action_weighted_clusters.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "RouteAction weighted_cluster has no valid clusters specified."); - } - route->action = std::move(action_weighted_clusters); - } else if (XdsRlsEnabled() && - envoy_config_route_v3_RouteAction_has_cluster_specifier_plugin( - route_action)) { - std::string plugin_name = UpbStringToStdString( - envoy_config_route_v3_RouteAction_cluster_specifier_plugin( - route_action)); - if (plugin_name.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "RouteAction cluster contains empty cluster specifier plugin name."); - } - auto it = cluster_specifier_plugin_map.find(plugin_name); - if (it == cluster_specifier_plugin_map.end()) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("RouteAction cluster contains cluster specifier plugin " - "name not configured: ", - plugin_name)); - } - if (it->second.empty()) *ignore_route = true; - route->action.emplace<XdsRouteConfigResource::Route::RouteAction:: - kClusterSpecifierPluginIndex>( - std::move(plugin_name)); - } else { - // No cluster or weighted_clusters or plugin found in RouteAction, ignore - // this route. - *ignore_route = true; - } - if (!*ignore_route) { - const envoy_config_route_v3_RouteAction_MaxStreamDuration* - max_stream_duration = - envoy_config_route_v3_RouteAction_max_stream_duration(route_action); - if (max_stream_duration != nullptr) { - const google_protobuf_Duration* duration = - envoy_config_route_v3_RouteAction_MaxStreamDuration_grpc_timeout_header_max( + ValidationErrors* errors) { + XdsRouteConfigResource::Route::RouteAction route_action; + // grpc_timeout_header_max or max_stream_duration + const auto* max_stream_duration = + envoy_config_route_v3_RouteAction_max_stream_duration(route_action_proto); + if (max_stream_duration != nullptr) { + ValidationErrors::ScopedField field(errors, ".max_stream_duration"); + const google_protobuf_Duration* duration = + envoy_config_route_v3_RouteAction_MaxStreamDuration_grpc_timeout_header_max( + max_stream_duration); + if (duration != nullptr) { + ValidationErrors::ScopedField field(errors, ".grpc_timeout_header_max"); + route_action.max_stream_duration = ParseDuration(duration, errors); + } else { + duration = + envoy_config_route_v3_RouteAction_MaxStreamDuration_max_stream_duration( max_stream_duration); - if (duration == nullptr) { - duration = - envoy_config_route_v3_RouteAction_MaxStreamDuration_max_stream_duration( - max_stream_duration); - } if (duration != nullptr) { - route->max_stream_duration = ParseDuration(duration); + ValidationErrors::ScopedField field(errors, ".max_stream_duration"); + route_action.max_stream_duration = ParseDuration(duration, errors); } } } - // Get HashPolicy from RouteAction + // hash_policy size_t size = 0; const envoy_config_route_v3_RouteAction_HashPolicy* const* hash_policies = - envoy_config_route_v3_RouteAction_hash_policy(route_action, &size); + envoy_config_route_v3_RouteAction_hash_policy(route_action_proto, &size); for (size_t i = 0; i < size; ++i) { - const envoy_config_route_v3_RouteAction_HashPolicy* hash_policy = - hash_policies[i]; + ValidationErrors::ScopedField field(errors, + absl::StrCat(".hash_policy[", i, "]")); + const auto* hash_policy = hash_policies[i]; XdsRouteConfigResource::Route::RouteAction::HashPolicy policy; policy.terminal = envoy_config_route_v3_RouteAction_HashPolicy_terminal(hash_policy); @@ -839,90 +789,264 @@ grpc_error_handle RouteActionParse( filter_state; if ((header = envoy_config_route_v3_RouteAction_HashPolicy_header( hash_policy)) != nullptr) { - policy.type = - XdsRouteConfigResource::Route::RouteAction::HashPolicy::Type::HEADER; - policy.header_name = UpbStringToStdString( + // header + ValidationErrors::ScopedField field(errors, ".header"); + XdsRouteConfigResource::Route::RouteAction::HashPolicy::Header + header_policy; + header_policy.header_name = UpbStringToStdString( envoy_config_route_v3_RouteAction_HashPolicy_Header_header_name( header)); - const struct envoy_type_matcher_v3_RegexMatchAndSubstitute* - regex_rewrite = - envoy_config_route_v3_RouteAction_HashPolicy_Header_regex_rewrite( - header); + if (header_policy.header_name.empty()) { + ValidationErrors::ScopedField field(errors, ".header_name"); + errors->AddError("must be non-empty"); + } + // regex_rewrite + const auto* regex_rewrite = + envoy_config_route_v3_RouteAction_HashPolicy_Header_regex_rewrite( + header); if (regex_rewrite != nullptr) { - const envoy_type_matcher_v3_RegexMatcher* regex_matcher = + ValidationErrors::ScopedField field(errors, ".regex_rewrite.pattern"); + const auto* pattern = envoy_type_matcher_v3_RegexMatchAndSubstitute_pattern( regex_rewrite); - if (regex_matcher == nullptr) { - gpr_log( - GPR_DEBUG, - "RouteAction HashPolicy contains policy specifier Header with " - "RegexMatchAndSubstitution but RegexMatcher pattern is " - "missing"); + if (pattern == nullptr) { + errors->AddError("field not present"); + continue; + } + ValidationErrors::ScopedField field2(errors, ".regex"); + std::string regex = UpbStringToStdString( + envoy_type_matcher_v3_RegexMatcher_regex(pattern)); + if (regex.empty()) { + errors->AddError("field not present"); continue; } RE2::Options options; - policy.regex = absl::make_unique<RE2>( - UpbStringToStdString( - envoy_type_matcher_v3_RegexMatcher_regex(regex_matcher)), - options); - if (!policy.regex->ok()) { - gpr_log( - GPR_DEBUG, - "RouteAction HashPolicy contains policy specifier Header with " - "RegexMatchAndSubstitution but RegexMatcher pattern does not " - "compile"); + header_policy.regex = std::make_unique<RE2>(regex, options); + if (!header_policy.regex->ok()) { + errors->AddError(absl::StrCat("errors compiling regex: ", + header_policy.regex->error())); continue; } - policy.regex_substitution = UpbStringToStdString( + header_policy.regex_substitution = UpbStringToStdString( envoy_type_matcher_v3_RegexMatchAndSubstitute_substitution( regex_rewrite)); } + policy.policy = std::move(header_policy); } else if ((filter_state = envoy_config_route_v3_RouteAction_HashPolicy_filter_state( hash_policy)) != nullptr) { + // filter_state std::string key = UpbStringToStdString( envoy_config_route_v3_RouteAction_HashPolicy_FilterState_key( filter_state)); - if (key == "io.grpc.channel_id") { - policy.type = XdsRouteConfigResource::Route::RouteAction::HashPolicy:: - Type::CHANNEL_ID; - } else { - gpr_log(GPR_DEBUG, - "RouteAction HashPolicy contains policy specifier " - "FilterState but " - "key is not io.grpc.channel_id."); - continue; - } + if (key != "io.grpc.channel_id") continue; + policy.policy = + XdsRouteConfigResource::Route::RouteAction::HashPolicy::ChannelId(); } else { - gpr_log(GPR_DEBUG, - "RouteAction HashPolicy contains unsupported policy specifier."); + // Unsupported hash policy type, ignore it. continue; } - route->hash_policies.emplace_back(std::move(policy)); + route_action.hash_policies.emplace_back(std::move(policy)); } // Get retry policy const envoy_config_route_v3_RetryPolicy* retry_policy = - envoy_config_route_v3_RouteAction_retry_policy(route_action); + envoy_config_route_v3_RouteAction_retry_policy(route_action_proto); if (retry_policy != nullptr) { - absl::optional<XdsRouteConfigResource::RetryPolicy> retry; - grpc_error_handle error = RetryPolicyParse(context, retry_policy, &retry); - if (error != GRPC_ERROR_NONE) return error; - route->retry_policy = retry; + ValidationErrors::ScopedField field(errors, ".retry_policy"); + route_action.retry_policy = RetryPolicyParse(context, retry_policy, errors); + } + // Parse cluster specifier, which is one of several options. + if (envoy_config_route_v3_RouteAction_has_cluster(route_action_proto)) { + // Cluster name. + std::string cluster_name = UpbStringToStdString( + envoy_config_route_v3_RouteAction_cluster(route_action_proto)); + if (cluster_name.empty()) { + ValidationErrors::ScopedField field(errors, ".cluster"); + errors->AddError("must be non-empty"); + } + route_action.action = + XdsRouteConfigResource::Route::RouteAction::ClusterName{ + std::move(cluster_name)}; + } else if (envoy_config_route_v3_RouteAction_has_weighted_clusters( + route_action_proto)) { + // WeightedClusters. + ValidationErrors::ScopedField field(errors, ".weighted_clusters"); + const envoy_config_route_v3_WeightedCluster* weighted_clusters_proto = + envoy_config_route_v3_RouteAction_weighted_clusters(route_action_proto); + GPR_ASSERT(weighted_clusters_proto != nullptr); + std::vector<XdsRouteConfigResource::Route::RouteAction::ClusterWeight> + action_weighted_clusters; + uint64_t total_weight = 0; + size_t clusters_size; + const envoy_config_route_v3_WeightedCluster_ClusterWeight* const* clusters = + envoy_config_route_v3_WeightedCluster_clusters(weighted_clusters_proto, + &clusters_size); + for (size_t i = 0; i < clusters_size; ++i) { + ValidationErrors::ScopedField field(errors, + absl::StrCat(".clusters[", i, "]")); + const auto* cluster_proto = clusters[i]; + XdsRouteConfigResource::Route::RouteAction::ClusterWeight cluster; + // typed_per_filter_config + { + ValidationErrors::ScopedField field(errors, ".typed_per_filter_config"); + cluster.typed_per_filter_config = ParseTypedPerFilterConfig< + envoy_config_route_v3_WeightedCluster_ClusterWeight, + envoy_config_route_v3_WeightedCluster_ClusterWeight_TypedPerFilterConfigEntry>( + context, cluster_proto, + envoy_config_route_v3_WeightedCluster_ClusterWeight_typed_per_filter_config_next, + envoy_config_route_v3_WeightedCluster_ClusterWeight_TypedPerFilterConfigEntry_key, + envoy_config_route_v3_WeightedCluster_ClusterWeight_TypedPerFilterConfigEntry_value, + errors); + } + // name + cluster.name = UpbStringToStdString( + envoy_config_route_v3_WeightedCluster_ClusterWeight_name( + cluster_proto)); + if (cluster.name.empty()) { + ValidationErrors::ScopedField field(errors, ".name"); + errors->AddError("must be non-empty"); + } + // weight + const google_protobuf_UInt32Value* weight_proto = + envoy_config_route_v3_WeightedCluster_ClusterWeight_weight( + cluster_proto); + if (weight_proto == nullptr) { + ValidationErrors::ScopedField field(errors, ".weight"); + errors->AddError("field not present"); + } else { + cluster.weight = google_protobuf_UInt32Value_value(weight_proto); + if (cluster.weight == 0) continue; + total_weight += cluster.weight; + } + // Add entry to WeightedClusters. + action_weighted_clusters.emplace_back(std::move(cluster)); + } + if (action_weighted_clusters.empty()) { + errors->AddError("no valid clusters specified"); + } else if (total_weight > std::numeric_limits<uint32_t>::max()) { + errors->AddError("sum of cluster weights exceeds uint32 max"); + } + route_action.action = std::move(action_weighted_clusters); + } else if (XdsRlsEnabled() && + envoy_config_route_v3_RouteAction_has_cluster_specifier_plugin( + route_action_proto)) { + // ClusterSpecifierPlugin + ValidationErrors::ScopedField field(errors, ".cluster_specifier_plugin"); + std::string plugin_name = UpbStringToStdString( + envoy_config_route_v3_RouteAction_cluster_specifier_plugin( + route_action_proto)); + if (plugin_name.empty()) { + errors->AddError("must be non-empty"); + return absl::nullopt; + } + auto it = cluster_specifier_plugin_map.find(plugin_name); + if (it == cluster_specifier_plugin_map.end()) { + errors->AddError(absl::StrCat("unknown cluster specifier plugin name \"", + plugin_name, "\"")); + } else { + // If the cluster specifier config is empty, that means that the + // plugin was unsupported but optional. In that case, skip this route. + if (it->second.empty()) return absl::nullopt; + } + route_action.action = + XdsRouteConfigResource::Route::RouteAction::ClusterSpecifierPluginName{ + std::move(plugin_name)}; + } else { + // Not a supported cluster specifier, so ignore this route. + return absl::nullopt; } - return GRPC_ERROR_NONE; + return route_action; +} + +absl::optional<XdsRouteConfigResource::Route> ParseRoute( + const XdsResourceType::DecodeContext& context, + const envoy_config_route_v3_Route* route_proto, + const absl::optional<XdsRouteConfigResource::RetryPolicy>& + virtual_host_retry_policy, + const XdsRouteConfigResource::ClusterSpecifierPluginMap& + cluster_specifier_plugin_map, + std::set<absl::string_view>* cluster_specifier_plugins_not_seen, + ValidationErrors* errors) { + XdsRouteConfigResource::Route route; + // Parse route match. + { + ValidationErrors::ScopedField field(errors, ".match"); + const auto* match = envoy_config_route_v3_Route_match(route_proto); + if (match == nullptr) { + errors->AddError("field not present"); + return absl::nullopt; + } + // Skip routes with query_parameters set. + size_t query_parameters_size; + static_cast<void>(envoy_config_route_v3_RouteMatch_query_parameters( + match, &query_parameters_size)); + if (query_parameters_size > 0) return absl::nullopt; + // Parse matchers. + auto path_matcher = RoutePathMatchParse(match, errors); + if (!path_matcher.has_value()) return absl::nullopt; + route.matchers.path_matcher = std::move(*path_matcher); + RouteHeaderMatchersParse(match, &route, errors); + RouteRuntimeFractionParse(match, &route, errors); + } + // Parse route action. + const envoy_config_route_v3_RouteAction* route_action_proto = + envoy_config_route_v3_Route_route(route_proto); + if (route_action_proto != nullptr) { + ValidationErrors::ScopedField field(errors, ".route"); + auto route_action = RouteActionParse(context, route_action_proto, + cluster_specifier_plugin_map, errors); + if (!route_action.has_value()) return absl::nullopt; + // If the route does not have a retry policy but the vhost does, + // use the vhost retry policy for this route. + if (!route_action->retry_policy.has_value()) { + route_action->retry_policy = virtual_host_retry_policy; + } + // Mark off plugins used in route action. + auto* cluster_specifier_action = absl::get_if< + XdsRouteConfigResource::Route::RouteAction::ClusterSpecifierPluginName>( + &route_action->action); + if (cluster_specifier_action != nullptr) { + cluster_specifier_plugins_not_seen->erase( + cluster_specifier_action->cluster_specifier_plugin_name); + } + route.action = std::move(*route_action); + } else if (envoy_config_route_v3_Route_has_non_forwarding_action( + route_proto)) { + route.action = XdsRouteConfigResource::Route::NonForwardingAction(); + } else { + // Leave route.action initialized to UnknownAction (its default). + } + // Parse typed_per_filter_config. + { + ValidationErrors::ScopedField field(errors, ".typed_per_filter_config"); + route.typed_per_filter_config = ParseTypedPerFilterConfig< + envoy_config_route_v3_Route, + envoy_config_route_v3_Route_TypedPerFilterConfigEntry>( + context, route_proto, + envoy_config_route_v3_Route_typed_per_filter_config_next, + envoy_config_route_v3_Route_TypedPerFilterConfigEntry_key, + envoy_config_route_v3_Route_TypedPerFilterConfigEntry_value, errors); + } + return route; } } // namespace -grpc_error_handle XdsRouteConfigResource::Parse( - const XdsEncodingContext& context, +XdsRouteConfigResource XdsRouteConfigResource::Parse( + const XdsResourceType::DecodeContext& context, const envoy_config_route_v3_RouteConfiguration* route_config, - XdsRouteConfigResource* rds_update) { - // Get the cluster spcifier plugins + ValidationErrors* errors) { + XdsRouteConfigResource rds_update; + // Get the cluster spcifier plugin map. if (XdsRlsEnabled()) { - grpc_error_handle error = - ClusterSpecifierPluginParse(context, route_config, rds_update); - if (error != GRPC_ERROR_NONE) return error; + rds_update.cluster_specifier_plugin_map = + ClusterSpecifierPluginParse(context, route_config, errors); + } + // Build a set of configured cluster_specifier_plugin names to make sure + // each is actually referenced by a route action. + std::set<absl::string_view> cluster_specifier_plugins_not_seen; + for (auto& plugin : rds_update.cluster_specifier_plugin_map) { + cluster_specifier_plugins_not_seen.emplace(plugin.first); } // Get the virtual hosts. size_t num_virtual_hosts; @@ -930,9 +1054,11 @@ grpc_error_handle XdsRouteConfigResource::Parse( envoy_config_route_v3_RouteConfiguration_virtual_hosts( route_config, &num_virtual_hosts); for (size_t i = 0; i < num_virtual_hosts; ++i) { - rds_update->virtual_hosts.emplace_back(); + ValidationErrors::ScopedField field( + errors, absl::StrCat(".virtual_hosts[", i, "]")); + rds_update.virtual_hosts.emplace_back(); XdsRouteConfigResource::VirtualHost& vhost = - rds_update->virtual_hosts.back(); + rds_update.virtual_hosts.back(); // Parse domains. size_t domain_size; upb_StringView const* domains = envoy_config_route_v3_VirtualHost_domains( @@ -940,25 +1066,28 @@ grpc_error_handle XdsRouteConfigResource::Parse( for (size_t j = 0; j < domain_size; ++j) { std::string domain_pattern = UpbStringToStdString(domains[j]); if (!XdsRouting::IsValidDomainPattern(domain_pattern)) { - return GRPC_ERROR_CREATE_FROM_CPP_STRING( - absl::StrCat("Invalid domain pattern \"", domain_pattern, "\".")); + ValidationErrors::ScopedField field(errors, + absl::StrCat(".domains[", j, "]")); + errors->AddError( + absl::StrCat("invalid domain pattern \"", domain_pattern, "\"")); } vhost.domains.emplace_back(std::move(domain_pattern)); } if (vhost.domains.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("VirtualHost has no domains"); + ValidationErrors::ScopedField field(errors, ".domains"); + errors->AddError("must be non-empty"); } // Parse typed_per_filter_config. - if (context.use_v3) { - grpc_error_handle error = ParseTypedPerFilterConfig< + { + ValidationErrors::ScopedField field(errors, ".typed_per_filter_config"); + vhost.typed_per_filter_config = ParseTypedPerFilterConfig< envoy_config_route_v3_VirtualHost, envoy_config_route_v3_VirtualHost_TypedPerFilterConfigEntry>( context, virtual_hosts[i], envoy_config_route_v3_VirtualHost_typed_per_filter_config_next, envoy_config_route_v3_VirtualHost_TypedPerFilterConfigEntry_key, envoy_config_route_v3_VirtualHost_TypedPerFilterConfigEntry_value, - &vhost.typed_per_filter_config); - if (error != GRPC_ERROR_NONE) return error; + errors); } // Parse retry policy. absl::optional<XdsRouteConfigResource::RetryPolicy> @@ -966,97 +1095,29 @@ grpc_error_handle XdsRouteConfigResource::Parse( const envoy_config_route_v3_RetryPolicy* retry_policy = envoy_config_route_v3_VirtualHost_retry_policy(virtual_hosts[i]); if (retry_policy != nullptr) { - grpc_error_handle error = - RetryPolicyParse(context, retry_policy, &virtual_host_retry_policy); - if (error != GRPC_ERROR_NONE) return error; + ValidationErrors::ScopedField field(errors, ".retry_policy"); + virtual_host_retry_policy = + RetryPolicyParse(context, retry_policy, errors); } // Parse routes. + ValidationErrors::ScopedField field2(errors, ".routes"); size_t num_routes; const envoy_config_route_v3_Route* const* routes = envoy_config_route_v3_VirtualHost_routes(virtual_hosts[i], &num_routes); - if (num_routes < 1) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "No route found in the virtual host."); - } - // Build a set of cluster_specifier_plugin configured to make sure each is - // actually referenced by a route action. - std::set<absl::string_view> cluster_specifier_plugins; - for (auto& plugin : rds_update->cluster_specifier_plugin_map) { - cluster_specifier_plugins.emplace(plugin.first); - } - // Loop over the whole list of routes for (size_t j = 0; j < num_routes; ++j) { - const envoy_config_route_v3_RouteMatch* match = - envoy_config_route_v3_Route_match(routes[j]); - if (match == nullptr) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("Match can't be null."); - } - size_t query_parameters_size; - static_cast<void>(envoy_config_route_v3_RouteMatch_query_parameters( - match, &query_parameters_size)); - if (query_parameters_size > 0) { - continue; - } - XdsRouteConfigResource::Route route; - bool ignore_route = false; - grpc_error_handle error = - RoutePathMatchParse(match, &route, &ignore_route); - if (error != GRPC_ERROR_NONE) return error; - if (ignore_route) continue; - error = RouteHeaderMatchersParse(match, &route); - if (error != GRPC_ERROR_NONE) return error; - error = RouteRuntimeFractionParse(match, &route); - if (error != GRPC_ERROR_NONE) return error; - if (envoy_config_route_v3_Route_has_route(routes[j])) { - route.action.emplace<XdsRouteConfigResource::Route::RouteAction>(); - auto& route_action = - absl::get<XdsRouteConfigResource::Route::RouteAction>(route.action); - error = RouteActionParse(context, routes[j], - rds_update->cluster_specifier_plugin_map, - &route_action, &ignore_route); - if (error != GRPC_ERROR_NONE) return error; - if (ignore_route) continue; - if (route_action.retry_policy == absl::nullopt && - retry_policy != nullptr) { - route_action.retry_policy = virtual_host_retry_policy; - } - // Mark off plugins used in route action. - std::string* cluster_specifier_action = - absl::get_if<XdsRouteConfigResource::Route::RouteAction:: - kClusterSpecifierPluginIndex>( - &route_action.action); - if (cluster_specifier_action != nullptr) { - cluster_specifier_plugins.erase(*cluster_specifier_action); - } - } else if (envoy_config_route_v3_Route_has_non_forwarding_action( - routes[j])) { - route.action - .emplace<XdsRouteConfigResource::Route::NonForwardingAction>(); - } - if (context.use_v3) { - grpc_error_handle error = ParseTypedPerFilterConfig< - envoy_config_route_v3_Route, - envoy_config_route_v3_Route_TypedPerFilterConfigEntry>( - context, routes[j], - envoy_config_route_v3_Route_typed_per_filter_config_next, - envoy_config_route_v3_Route_TypedPerFilterConfigEntry_key, - envoy_config_route_v3_Route_TypedPerFilterConfigEntry_value, - &route.typed_per_filter_config); - if (error != GRPC_ERROR_NONE) return error; - } - vhost.routes.emplace_back(std::move(route)); - } - if (vhost.routes.empty()) { - return GRPC_ERROR_CREATE_FROM_STATIC_STRING("No valid routes specified."); - } - // For plugins not used in route action, delete from the update to prevent - // further use. - for (auto& unused_plugin : cluster_specifier_plugins) { - rds_update->cluster_specifier_plugin_map.erase( - std::string(unused_plugin)); + ValidationErrors::ScopedField field(errors, absl::StrCat("[", j, "]")); + auto route = ParseRoute(context, routes[j], virtual_host_retry_policy, + rds_update.cluster_specifier_plugin_map, + &cluster_specifier_plugins_not_seen, errors); + if (route.has_value()) vhost.routes.emplace_back(std::move(*route)); } } - return GRPC_ERROR_NONE; + // For cluster specifier plugins that were not used in any route action, + // delete them from the update, since they will never be used. + for (auto& unused_plugin : cluster_specifier_plugins_not_seen) { + rds_update.cluster_specifier_plugin_map.erase(std::string(unused_plugin)); + } + return rds_update; } // @@ -1066,7 +1127,7 @@ grpc_error_handle XdsRouteConfigResource::Parse( namespace { void MaybeLogRouteConfiguration( - const XdsEncodingContext& context, + const XdsResourceType::DecodeContext& context, const envoy_config_route_v3_RouteConfiguration* route_config) { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer) && gpr_should_log(GPR_LOG_SEVERITY_DEBUG)) { @@ -1081,42 +1142,43 @@ void MaybeLogRouteConfiguration( } // namespace -absl::StatusOr<XdsResourceType::DecodeResult> -XdsRouteConfigResourceType::Decode(const XdsEncodingContext& context, - absl::string_view serialized_resource, - bool /*is_v2*/) const { +XdsResourceType::DecodeResult XdsRouteConfigResourceType::Decode( + const XdsResourceType::DecodeContext& context, + absl::string_view serialized_resource) const { + DecodeResult result; // Parse serialized proto. auto* resource = envoy_config_route_v3_RouteConfiguration_parse( serialized_resource.data(), serialized_resource.size(), context.arena); if (resource == nullptr) { - return absl::InvalidArgumentError( - "Can't parse RouteConfiguration resource."); + result.resource = + absl::InvalidArgumentError("Can't parse RouteConfiguration resource."); + return result; } MaybeLogRouteConfiguration(context, resource); // Validate resource. - DecodeResult result; result.name = UpbStringToStdString( envoy_config_route_v3_RouteConfiguration_name(resource)); - auto route_config_data = absl::make_unique<ResourceDataSubclass>(); - grpc_error_handle error = XdsRouteConfigResource::Parse( - context, resource, &route_config_data->resource); - if (error != GRPC_ERROR_NONE) { - std::string error_str = grpc_error_std_string(error); - GRPC_ERROR_UNREF(error); + ValidationErrors errors; + auto rds_update = XdsRouteConfigResource::Parse(context, resource, &errors); + if (!errors.ok()) { + absl::Status status = + errors.status(absl::StatusCode::kInvalidArgument, + "errors validating RouteConfiguration resource"); if (GRPC_TRACE_FLAG_ENABLED(*context.tracer)) { gpr_log(GPR_ERROR, "[xds_client %p] invalid RouteConfiguration %s: %s", - context.client, result.name.c_str(), error_str.c_str()); + context.client, result.name->c_str(), status.ToString().c_str()); } - result.resource = absl::InvalidArgumentError(error_str); + result.resource = std::move(status); } else { if (GRPC_TRACE_FLAG_ENABLED(*context.tracer)) { gpr_log(GPR_INFO, "[xds_client %p] parsed RouteConfiguration %s: %s", - context.client, result.name.c_str(), - route_config_data->resource.ToString().c_str()); + context.client, result.name->c_str(), + rds_update.ToString().c_str()); } - result.resource = std::move(route_config_data); + result.resource = + std::make_unique<XdsRouteConfigResource>(std::move(rds_update)); } - return std::move(result); + return result; } } // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_route_config.h b/grpc/src/core/ext/xds/xds_route_config.h index dd719b0c..09c9c912 100644 --- a/grpc/src/core/ext/xds/xds_route_config.h +++ b/grpc/src/core/ext/xds/xds_route_config.h @@ -14,37 +14,48 @@ // limitations under the License. // -#ifndef GRPC_CORE_EXT_XDS_XDS_ROUTE_CONFIG_H -#define GRPC_CORE_EXT_XDS_XDS_ROUTE_CONFIG_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_ROUTE_CONFIG_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_ROUTE_CONFIG_H #include <grpc/support/port_platform.h> +#include <stdint.h> + +#include <algorithm> #include <map> +#include <memory> #include <string> #include <vector> +#include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/variant.h" #include "envoy/config/route/v3/route.upb.h" #include "envoy/config/route/v3/route.upbdefs.h" #include "re2/re2.h" +#include "upb/reflection/def.h" +#include "src/core/ext/xds/xds_bootstrap_grpc.h" #include "src/core/ext/xds/xds_client.h" #include "src/core/ext/xds/xds_cluster_specifier_plugin.h" -#include "src/core/ext/xds/xds_common_types.h" #include "src/core/ext/xds/xds_http_filters.h" +#include "src/core/ext/xds/xds_resource_type.h" #include "src/core/ext/xds/xds_resource_type_impl.h" #include "src/core/lib/channel/status_util.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/gprpp/validation_errors.h" #include "src/core/lib/matchers/matchers.h" namespace grpc_core { -bool XdsRbacEnabled(); - -struct XdsRouteConfigResource { +struct XdsRouteConfigResource : public XdsResourceType::ResourceData { using TypedPerFilterConfig = std::map<std::string, XdsHttpFilterImpl::FilterConfig>; + using ClusterSpecifierPluginMap = + std::map<std::string /*cluster_specifier_plugin_name*/, + std::string /*LB policy config*/>; + struct RetryPolicy { internal::StatusCodeSet retry_on; uint32_t num_retries; @@ -91,28 +102,46 @@ struct XdsRouteConfigResource { struct RouteAction { struct HashPolicy { - enum Type { HEADER, CHANNEL_ID }; - Type type; - bool terminal = false; - // Fields used for type HEADER. - std::string header_name; - std::unique_ptr<RE2> regex = nullptr; - std::string regex_substitution; + struct Header { + std::string header_name; + std::unique_ptr<RE2> regex; + std::string regex_substitution; + + Header() = default; - HashPolicy() {} + // Copyable. + Header(const Header& other); + Header& operator=(const Header& other); - // Copyable. - HashPolicy(const HashPolicy& other); - HashPolicy& operator=(const HashPolicy& other); + // Movable. + Header(Header&& other) noexcept; + Header& operator=(Header&& other) noexcept; - // Moveable. - HashPolicy(HashPolicy&& other) noexcept; - HashPolicy& operator=(HashPolicy&& other) noexcept; + bool operator==(const Header& other) const; + std::string ToString() const; + }; - bool operator==(const HashPolicy& other) const; + struct ChannelId { + bool operator==(const ChannelId&) const { return true; } + }; + + absl::variant<Header, ChannelId> policy; + bool terminal = false; + + bool operator==(const HashPolicy& other) const { + return policy == other.policy && terminal == other.terminal; + } std::string ToString() const; }; + struct ClusterName { + std::string cluster_name; + + bool operator==(const ClusterName& other) const { + return cluster_name == other.cluster_name; + } + }; + struct ClusterWeight { std::string name; uint32_t weight; @@ -125,14 +154,21 @@ struct XdsRouteConfigResource { std::string ToString() const; }; + struct ClusterSpecifierPluginName { + std::string cluster_specifier_plugin_name; + + bool operator==(const ClusterSpecifierPluginName& other) const { + return cluster_specifier_plugin_name == + other.cluster_specifier_plugin_name; + } + }; + std::vector<HashPolicy> hash_policies; absl::optional<RetryPolicy> retry_policy; // Action for this route. - static constexpr size_t kClusterIndex = 0; - static constexpr size_t kWeightedClustersIndex = 1; - static constexpr size_t kClusterSpecifierPluginIndex = 2; - absl::variant<std::string, std::vector<ClusterWeight>, std::string> + absl::variant<ClusterName, std::vector<ClusterWeight>, + ClusterSpecifierPluginName> action; // Storing the timeout duration from route action: // RouteAction.max_stream_duration.grpc_timeout_header_max or @@ -176,9 +212,7 @@ struct XdsRouteConfigResource { }; std::vector<VirtualHost> virtual_hosts; - std::map<std::string /*cluster_specifier_plugin_name*/, - std::string /*LB policy config*/> - cluster_specifier_plugin_map; + ClusterSpecifierPluginMap cluster_specifier_plugin_map; bool operator==(const XdsRouteConfigResource& other) const { return virtual_hosts == other.virtual_hosts && @@ -186,10 +220,10 @@ struct XdsRouteConfigResource { } std::string ToString() const; - static grpc_error_handle Parse( - const XdsEncodingContext& context, + static XdsRouteConfigResource Parse( + const XdsResourceType::DecodeContext& context, const envoy_config_route_v3_RouteConfiguration* route_config, - XdsRouteConfigResource* rds_update); + ValidationErrors* errors); }; class XdsRouteConfigResourceType @@ -199,20 +233,20 @@ class XdsRouteConfigResourceType absl::string_view type_url() const override { return "envoy.config.route.v3.RouteConfiguration"; } - absl::string_view v2_type_url() const override { - return "envoy.api.v2.RouteConfiguration"; - } - absl::StatusOr<DecodeResult> Decode(const XdsEncodingContext& context, - absl::string_view serialized_resource, - bool /*is_v2*/) const override; + DecodeResult Decode(const XdsResourceType::DecodeContext& context, + absl::string_view serialized_resource) const override; - void InitUpbSymtab(upb_DefPool* symtab) const override { + void InitUpbSymtab(XdsClient* xds_client, + upb_DefPool* symtab) const override { envoy_config_route_v3_RouteConfiguration_getmsgdef(symtab); - XdsClusterSpecifierPluginRegistry::PopulateSymtab(symtab); + const auto& cluster_specifier_plugin_registry = + static_cast<const GrpcXdsBootstrap&>(xds_client->bootstrap()) + .cluster_specifier_plugin_registry(); + cluster_specifier_plugin_registry.PopulateSymtab(symtab); } }; } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_ROUTE_CONFIG_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_ROUTE_CONFIG_H diff --git a/grpc/src/core/ext/xds/xds_routing.cc b/grpc/src/core/ext/xds/xds_routing.cc index f075bfc0..5d56b2bf 100644 --- a/grpc/src/core/ext/xds/xds_routing.cc +++ b/grpc/src/core/ext/xds/xds_routing.cc @@ -20,7 +20,23 @@ #include "src/core/ext/xds/xds_routing.h" +#include <stdint.h> +#include <stdlib.h> + +#include <algorithm> #include <cctype> +#include <utility> + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/str_cat.h" + +#include <grpc/support/log.h> + +#include "src/core/ext/xds/xds_http_filters.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/matchers/matchers.h" namespace grpc_core { @@ -202,22 +218,23 @@ const XdsHttpFilterImpl::FilterConfig* FindFilterConfigOverride( } // namespace -XdsRouting::GeneratePerHttpFilterConfigsResult +absl::StatusOr<XdsRouting::GeneratePerHttpFilterConfigsResult> XdsRouting::GeneratePerHTTPFilterConfigs( + const XdsHttpFilterRegistry& http_filter_registry, const std::vector<XdsListenerResource::HttpConnectionManager::HttpFilter>& http_filters, const XdsRouteConfigResource::VirtualHost& vhost, const XdsRouteConfigResource::Route& route, const XdsRouteConfigResource::Route::RouteAction::ClusterWeight* cluster_weight, - grpc_channel_args* args) { + const ChannelArgs& args) { GeneratePerHttpFilterConfigsResult result; result.args = args; for (const auto& http_filter : http_filters) { // Find filter. This is guaranteed to succeed, because it's checked // at config validation time in the XdsApi code. const XdsHttpFilterImpl* filter_impl = - XdsHttpFilterRegistry::GetFilterForType( + http_filter_registry.GetFilterForType( http_filter.config.config_proto_type_name); GPR_ASSERT(filter_impl != nullptr); // If there is not actually any C-core filter associated with this @@ -231,15 +248,12 @@ XdsRouting::GeneratePerHTTPFilterConfigs( FindFilterConfigOverride(http_filter.name, vhost, route, cluster_weight); // Generate service config for filter. - auto method_config_field = - filter_impl->GenerateServiceConfig(http_filter.config, config_override); + auto method_config_field = filter_impl->GenerateServiceConfig( + http_filter.config, config_override, http_filter.name); if (!method_config_field.ok()) { - grpc_channel_args_destroy(result.args); - result.args = nullptr; - result.error = GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat( + return absl::FailedPreconditionError(absl::StrCat( "failed to generate method config for HTTP filter ", http_filter.name, ": ", method_config_field.status().ToString())); - break; } result.per_filter_configs[method_config_field->service_config_field_name] .push_back(method_config_field->element); diff --git a/grpc/src/core/ext/xds/xds_routing.h b/grpc/src/core/ext/xds/xds_routing.h index 18df4ce4..67f2e98c 100644 --- a/grpc/src/core/ext/xds/xds_routing.h +++ b/grpc/src/core/ext/xds/xds_routing.h @@ -16,20 +16,25 @@ // // -#ifndef GRPC_CORE_EXT_XDS_XDS_ROUTING_H -#define GRPC_CORE_EXT_XDS_XDS_ROUTING_H +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_ROUTING_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_ROUTING_H #include <grpc/support/port_platform.h> +#include <stddef.h> + +#include <map> +#include <string> #include <vector> +#include "absl/status/statusor.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" -#include <grpc/support/log.h> - +#include "src/core/ext/xds/xds_http_filters.h" #include "src/core/ext/xds/xds_listener.h" #include "src/core/ext/xds/xds_route_config.h" -#include "src/core/lib/matchers/matchers.h" +#include "src/core/lib/channel/channel_args.h" #include "src/core/lib/transport/metadata_batch.h" namespace grpc_core { @@ -78,24 +83,24 @@ class XdsRouting { std::string* concatenated_value); struct GeneratePerHttpFilterConfigsResult { - // Map of field name to list of elements for that field + // Map of service config field name to list of elements for that field. std::map<std::string, std::vector<std::string>> per_filter_configs; - grpc_error_handle error = GRPC_ERROR_NONE; - // Guaranteed to be nullptr if error is GRPC_ERROR_NONE - grpc_channel_args* args = nullptr; + ChannelArgs args; }; // Generates a map of per_filter_configs. \a args is consumed. - static GeneratePerHttpFilterConfigsResult GeneratePerHTTPFilterConfigs( + static absl::StatusOr<GeneratePerHttpFilterConfigsResult> + GeneratePerHTTPFilterConfigs( + const XdsHttpFilterRegistry& http_filter_registry, const std::vector<XdsListenerResource::HttpConnectionManager::HttpFilter>& http_filters, const XdsRouteConfigResource::VirtualHost& vhost, const XdsRouteConfigResource::Route& route, const XdsRouteConfigResource::Route::RouteAction::ClusterWeight* cluster_weight, - grpc_channel_args* args); + const ChannelArgs& args); }; } // namespace grpc_core -#endif // GRPC_CORE_EXT_XDS_XDS_ROUTING_H +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_ROUTING_H diff --git a/grpc/src/core/ext/xds/xds_server_config_fetcher.cc b/grpc/src/core/ext/xds/xds_server_config_fetcher.cc index 9bae129d..2b457041 100644 --- a/grpc/src/core/ext/xds/xds_server_config_fetcher.cc +++ b/grpc/src/core/ext/xds/xds_server_config_fetcher.cc @@ -18,30 +18,74 @@ #include <grpc/support/port_platform.h> +#include <string.h> + +#include <algorithm> +#include <map> +#include <memory> +#include <set> +#include <string> +#include <utility> +#include <vector> + +#include "absl/base/thread_annotations.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/strings/match.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" #include "absl/strings/str_join.h" #include "absl/strings/str_replace.h" +#include "absl/strings/string_view.h" +#include "absl/types/optional.h" +#include "absl/types/variant.h" + +#include <grpc/grpc.h> +#include <grpc/grpc_security.h> +#include <grpc/slice.h> +#include <grpc/status.h> +#include <grpc/support/log.h> #include "src/core/ext/filters/server_config_selector/server_config_selector.h" #include "src/core/ext/filters/server_config_selector/server_config_selector_filter.h" +#include "src/core/ext/xds/certificate_provider_store.h" +#include "src/core/ext/xds/xds_bootstrap_grpc.h" #include "src/core/ext/xds/xds_certificate_provider.h" #include "src/core/ext/xds/xds_channel_stack_modifier.h" -#include "src/core/ext/xds/xds_client.h" +#include "src/core/ext/xds/xds_client_grpc.h" +#include "src/core/ext/xds/xds_common_types.h" +#include "src/core/ext/xds/xds_http_filters.h" #include "src/core/ext/xds/xds_listener.h" #include "src/core/ext/xds/xds_route_config.h" #include "src/core/ext/xds/xds_routing.h" #include "src/core/lib/address_utils/parse_address.h" #include "src/core/lib/address_utils/sockaddr_utils.h" #include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_args_preconditioning.h" +#include "src/core/lib/channel/channel_fwd.h" #include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/debug/trace.h" +#include "src/core/lib/gprpp/debug_location.h" #include "src/core/lib/gprpp/host_port.h" +#include "src/core/lib/gprpp/match.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/sync.h" +#include "src/core/lib/gprpp/unique_type_name.h" +#include "src/core/lib/iomgr/endpoint.h" +#include "src/core/lib/iomgr/exec_ctx.h" +#include "src/core/lib/iomgr/iomgr_fwd.h" +#include "src/core/lib/iomgr/resolved_address.h" #include "src/core/lib/iomgr/sockaddr.h" #include "src/core/lib/iomgr/socket_utils.h" +#include "src/core/lib/security/credentials/credentials.h" +#include "src/core/lib/security/credentials/tls/grpc_tls_certificate_distributor.h" +#include "src/core/lib/security/credentials/tls/grpc_tls_certificate_provider.h" #include "src/core/lib/security/credentials/xds/xds_credentials.h" +#include "src/core/lib/service_config/service_config.h" #include "src/core/lib/service_config/service_config_impl.h" -#include "src/core/lib/slice/slice_internal.h" #include "src/core/lib/surface/api_trace.h" #include "src/core/lib/surface/server.h" -#include "src/core/lib/transport/error_utils.h" +#include "src/core/lib/transport/metadata_batch.h" #include "src/core/lib/uri/uri_parser.h" namespace grpc_core { @@ -54,9 +98,13 @@ TraceFlag grpc_xds_server_config_fetcher_trace(false, // listeners from the xDS control plane. class XdsServerConfigFetcher : public grpc_server_config_fetcher { public: - XdsServerConfigFetcher(RefCountedPtr<XdsClient> xds_client, + XdsServerConfigFetcher(RefCountedPtr<GrpcXdsClient> xds_client, grpc_server_xds_status_notifier notifier); + ~XdsServerConfigFetcher() override { + xds_client_.reset(DEBUG_LOCATION, "XdsServerConfigFetcher"); + } + void StartWatch(std::string listening_address, std::unique_ptr<grpc_server_config_fetcher::WatcherInterface> watcher) override; @@ -72,7 +120,7 @@ class XdsServerConfigFetcher : public grpc_server_config_fetcher { private: class ListenerWatcher; - const RefCountedPtr<XdsClient> xds_client_; + RefCountedPtr<GrpcXdsClient> xds_client_; const grpc_server_xds_status_notifier serving_status_notifier_; Mutex mu_; std::map<grpc_server_config_fetcher::WatcherInterface*, ListenerWatcher*> @@ -92,12 +140,16 @@ class XdsServerConfigFetcher : public grpc_server_config_fetcher { class XdsServerConfigFetcher::ListenerWatcher : public XdsListenerResourceType::WatcherInterface { public: - ListenerWatcher(RefCountedPtr<XdsClient> xds_client, + ListenerWatcher(RefCountedPtr<GrpcXdsClient> xds_client, std::unique_ptr<grpc_server_config_fetcher::WatcherInterface> server_config_watcher, grpc_server_xds_status_notifier serving_status_notifier, std::string listening_address); + ~ListenerWatcher() override { + xds_client_.reset(DEBUG_LOCATION, "ListenerWatcher"); + } + void OnResourceChanged(XdsListenerResource listener) override; void OnError(absl::Status status) override; @@ -124,7 +176,7 @@ class XdsServerConfigFetcher::ListenerWatcher FilterChainMatchManager* filter_chain_match_manager) ABSL_EXCLUSIVE_LOCKS_REQUIRED(&mu_); - const RefCountedPtr<XdsClient> xds_client_; + RefCountedPtr<GrpcXdsClient> xds_client_; const std::unique_ptr<grpc_server_config_fetcher::WatcherInterface> server_config_watcher_; const grpc_server_xds_status_notifier serving_status_notifier_; @@ -144,13 +196,17 @@ class XdsServerConfigFetcher::ListenerWatcher class XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager : public grpc_server_config_fetcher::ConnectionManager { public: - FilterChainMatchManager(RefCountedPtr<XdsClient> xds_client, + FilterChainMatchManager(RefCountedPtr<GrpcXdsClient> xds_client, XdsListenerResource::FilterChainMap filter_chain_map, absl::optional<XdsListenerResource::FilterChainData> default_filter_chain); - absl::StatusOr<grpc_channel_args*> UpdateChannelArgsForConnection( - grpc_channel_args* args, grpc_endpoint* tcp) override; + ~FilterChainMatchManager() override { + xds_client_.reset(DEBUG_LOCATION, "FilterChainMatchManager"); + } + + absl::StatusOr<ChannelArgs> UpdateChannelArgsForConnection( + const ChannelArgs& args, grpc_endpoint* tcp) override; void Orphan() override; @@ -198,7 +254,7 @@ class XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager void OnError(const std::string& resource_name, absl::Status status); void OnResourceDoesNotExist(const std::string& resource_name); - RefCountedPtr<XdsClient> xds_client_; + RefCountedPtr<GrpcXdsClient> xds_client_; // This ref is only kept around till the FilterChainMatchManager becomes // ready. RefCountedPtr<ListenerWatcher> listener_watcher_; @@ -254,12 +310,14 @@ class XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: XdsServerConfigSelector : public ServerConfigSelector { public: static absl::StatusOr<RefCountedPtr<XdsServerConfigSelector>> Create( + const XdsHttpFilterRegistry& http_filter_registry, XdsRouteConfigResource rds_update, const std::vector<XdsListenerResource::HttpConnectionManager::HttpFilter>& http_filters); ~XdsServerConfigSelector() override = default; - CallConfig GetCallConfig(grpc_metadata_batch* metadata) override; + absl::StatusOr<CallConfig> GetCallConfig( + grpc_metadata_batch* metadata) override; private: struct VirtualHost { @@ -317,12 +375,18 @@ class XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: : public ServerConfigSelectorProvider { public: StaticXdsServerConfigSelectorProvider( + RefCountedPtr<GrpcXdsClient> xds_client, absl::StatusOr<XdsRouteConfigResource> static_resource, std::vector<XdsListenerResource::HttpConnectionManager::HttpFilter> http_filters) - : static_resource_(std::move(static_resource)), + : xds_client_(std::move(xds_client)), + static_resource_(std::move(static_resource)), http_filters_(std::move(http_filters)) {} + ~StaticXdsServerConfigSelectorProvider() override { + xds_client_.reset(DEBUG_LOCATION, "StaticXdsServerConfigSelectorProvider"); + } + absl::StatusOr<RefCountedPtr<ServerConfigSelector>> Watch( std::unique_ptr<ServerConfigSelectorProvider::ServerConfigSelectorWatcher> watcher) override { @@ -331,8 +395,10 @@ class XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: if (!static_resource_.ok()) { return static_resource_.status(); } - return XdsServerConfigSelector::Create(static_resource_.value(), - http_filters_); + return XdsServerConfigSelector::Create( + static_cast<const GrpcXdsBootstrap&>(xds_client_->bootstrap()) + .http_filter_registry(), + static_resource_.value(), http_filters_); } void Orphan() override {} @@ -340,6 +406,7 @@ class XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: void CancelWatch() override { watcher_.reset(); } private: + RefCountedPtr<GrpcXdsClient> xds_client_; absl::StatusOr<XdsRouteConfigResource> static_resource_; std::vector<XdsListenerResource::HttpConnectionManager::HttpFilter> http_filters_; @@ -354,11 +421,15 @@ class XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: : public ServerConfigSelectorProvider { public: DynamicXdsServerConfigSelectorProvider( - RefCountedPtr<XdsClient> xds_client, std::string resource_name, + RefCountedPtr<GrpcXdsClient> xds_client, std::string resource_name, absl::StatusOr<XdsRouteConfigResource> initial_resource, std::vector<XdsListenerResource::HttpConnectionManager::HttpFilter> http_filters); + ~DynamicXdsServerConfigSelectorProvider() override { + xds_client_.reset(DEBUG_LOCATION, "DynamicXdsServerConfigSelectorProvider"); + } + void Orphan() override; absl::StatusOr<RefCountedPtr<ServerConfigSelector>> Watch( @@ -373,7 +444,7 @@ class XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: void OnError(absl::Status status); void OnResourceDoesNotExist(); - RefCountedPtr<XdsClient> xds_client_; + RefCountedPtr<GrpcXdsClient> xds_client_; std::string resource_name_; std::vector<XdsListenerResource::HttpConnectionManager::HttpFilter> http_filters_; @@ -411,7 +482,7 @@ class XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: // XdsServerConfigFetcher::XdsServerConfigFetcher( - RefCountedPtr<XdsClient> xds_client, + RefCountedPtr<GrpcXdsClient> xds_client, grpc_server_xds_status_notifier notifier) : xds_client_(std::move(xds_client)), serving_status_notifier_(notifier) { GPR_ASSERT(xds_client_ != nullptr); @@ -433,13 +504,14 @@ void XdsServerConfigFetcher::StartWatch( std::unique_ptr<grpc_server_config_fetcher::WatcherInterface> watcher) { grpc_server_config_fetcher::WatcherInterface* watcher_ptr = watcher.get(); auto listener_watcher = MakeRefCounted<ListenerWatcher>( - xds_client_, std::move(watcher), serving_status_notifier_, - listening_address); + xds_client_->Ref(DEBUG_LOCATION, "ListenerWatcher"), std::move(watcher), + serving_status_notifier_, listening_address); auto* listener_watcher_ptr = listener_watcher.get(); XdsListenerResourceType::StartWatch( xds_client_.get(), ListenerResourceName( - xds_client_->bootstrap().server_listener_resource_name_template(), + static_cast<const GrpcXdsBootstrap&>(xds_client_->bootstrap()) + .server_listener_resource_name_template(), listening_address), std::move(listener_watcher)); MutexLock lock(&mu_); @@ -455,7 +527,8 @@ void XdsServerConfigFetcher::CancelWatch( XdsListenerResourceType::CancelWatch( xds_client_.get(), ListenerResourceName( - xds_client_->bootstrap().server_listener_resource_name_template(), + static_cast<const GrpcXdsBootstrap&>(xds_client_->bootstrap()) + .server_listener_resource_name_template(), it->second->listening_address()), it->second, false /* delay_unsubscription */); listener_watchers_.erase(it); @@ -467,7 +540,7 @@ void XdsServerConfigFetcher::CancelWatch( // XdsServerConfigFetcher::ListenerWatcher::ListenerWatcher( - RefCountedPtr<XdsClient> xds_client, + RefCountedPtr<GrpcXdsClient> xds_client, std::unique_ptr<grpc_server_config_fetcher::WatcherInterface> server_config_watcher, grpc_server_xds_status_notifier serving_status_notifier, @@ -484,15 +557,24 @@ void XdsServerConfigFetcher::ListenerWatcher::OnResourceChanged( "[ListenerWatcher %p] Received LDS update from xds client %p: %s", this, xds_client_.get(), listener.ToString().c_str()); } - if (listener.address != listening_address_) { + auto* tcp_listener = + absl::get_if<XdsListenerResource::TcpListener>(&listener.listener); + if (tcp_listener == nullptr) { + MutexLock lock(&mu_); + OnFatalError( + absl::FailedPreconditionError("LDS resource is not a TCP listener")); + return; + } + if (tcp_listener->address != listening_address_) { MutexLock lock(&mu_); OnFatalError(absl::FailedPreconditionError( "Address in LDS update does not match listening address")); return; } auto new_filter_chain_match_manager = MakeRefCounted<FilterChainMatchManager>( - xds_client_, std::move(listener.filter_chain_map), - std::move(listener.default_filter_chain)); + xds_client_->Ref(DEBUG_LOCATION, "FilterChainMatchManager"), + std::move(tcp_listener->filter_chain_map), + std::move(tcp_listener->default_filter_chain)); MutexLock lock(&mu_); if (filter_chain_match_manager_ == nullptr || !(new_filter_chain_match_manager->filter_chain_map() == @@ -501,12 +583,7 @@ void XdsServerConfigFetcher::ListenerWatcher::OnResourceChanged( filter_chain_match_manager_->default_filter_chain())) { pending_filter_chain_match_manager_ = std::move(new_filter_chain_match_manager); - if (XdsRbacEnabled()) { - pending_filter_chain_match_manager_->StartRdsWatch(Ref()); - } else { - PendingFilterChainMatchManagerReadyLocked( - pending_filter_chain_match_manager_.get()); - } + pending_filter_chain_match_manager_->StartRdsWatch(Ref()); } } @@ -594,7 +671,7 @@ void XdsServerConfigFetcher::ListenerWatcher:: XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: FilterChainMatchManager( - RefCountedPtr<XdsClient> xds_client, + RefCountedPtr<GrpcXdsClient> xds_client, XdsListenerResource::FilterChainMap filter_chain_map, absl::optional<XdsListenerResource::FilterChainData> default_filter_chain) @@ -613,32 +690,25 @@ void XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: for (const auto& source_type : destination_ip.source_types_array) { for (const auto& source_ip : source_type) { for (const auto& source_port_pair : source_ip.ports_map) { - if (!source_port_pair.second.data->http_connection_manager - .route_config_name.empty()) { - resource_names.insert( - source_port_pair.second.data->http_connection_manager - .route_config_name); - } - filter_chain_data_set.insert(source_port_pair.second.data.get()); + auto* filter_chain_data = source_port_pair.second.data.get(); + const auto* rds_name = absl::get_if<std::string>( + &filter_chain_data->http_connection_manager.route_config); + if (rds_name != nullptr) resource_names.insert(*rds_name); + filter_chain_data_set.insert(filter_chain_data); } } } } if (default_filter_chain_.has_value()) { - if (!default_filter_chain_->http_connection_manager.route_config_name - .empty()) { - resource_names.insert( - default_filter_chain_->http_connection_manager.route_config_name); - } - std::reverse( - default_filter_chain_->http_connection_manager.http_filters.begin(), - default_filter_chain_->http_connection_manager.http_filters.end()); + auto& hcm = default_filter_chain_->http_connection_manager; + const auto* rds_name = absl::get_if<std::string>(&hcm.route_config); + if (rds_name != nullptr) resource_names.insert(*rds_name); + std::reverse(hcm.http_filters.begin(), hcm.http_filters.end()); } // Reverse the lists of HTTP filters in all the filter chains for (auto* filter_chain_data : filter_chain_data_set) { - std::reverse( - filter_chain_data->http_connection_manager.http_filters.begin(), - filter_chain_data->http_connection_manager.http_filters.end()); + auto& hcm = filter_chain_data->http_connection_manager; + std::reverse(hcm.http_filters.begin(), hcm.http_filters.end()); } // Start watching on referenced RDS resources struct WatcherToStart { @@ -897,13 +967,10 @@ const XdsListenerResource::FilterChainData* FindFilterChainDataForSourceType( if (!SplitHostPort(source_uri->path(), &host, &port)) { return nullptr; } - grpc_resolved_address source_addr; - grpc_error_handle error = grpc_string_to_sockaddr( - &source_addr, host.c_str(), 0 /* port doesn't matter here */); - if (error != GRPC_ERROR_NONE) { - gpr_log(GPR_DEBUG, "Could not parse string to socket address: %s", - host.c_str()); - GRPC_ERROR_UNREF(error); + auto source_addr = StringToSockaddr(host, 0); // Port doesn't matter here. + if (!source_addr.ok()) { + gpr_log(GPR_DEBUG, "Could not parse \"%s\" as socket address: %s", + host.c_str(), source_addr.status().ToString().c_str()); return nullptr; } // Use kAny only if kSameIporLoopback and kExternal are empty @@ -917,20 +984,20 @@ const XdsListenerResource::FilterChainData* FindFilterChainDataForSourceType( return FindFilterChainDataForSourceIp( source_types_array[static_cast<int>( XdsListenerResource::FilterChainMap::ConnectionSourceType::kAny)], - &source_addr, port); + &*source_addr, port); } - if (IsLoopbackIp(&source_addr) || host == destination_ip) { + if (IsLoopbackIp(&*source_addr) || host == destination_ip) { return FindFilterChainDataForSourceIp( source_types_array[static_cast<int>( XdsListenerResource::FilterChainMap::ConnectionSourceType:: kSameIpOrLoopback)], - &source_addr, port); + &*source_addr, port); } else { return FindFilterChainDataForSourceIp( source_types_array[static_cast<int>( XdsListenerResource::FilterChainMap::ConnectionSourceType:: kExternal)], - &source_addr, port); + &*source_addr, port); } } @@ -948,13 +1015,11 @@ const XdsListenerResource::FilterChainData* FindFilterChainDataForDestinationIp( if (!SplitHostPort(destination_uri->path(), &host, &port)) { return nullptr; } - grpc_resolved_address destination_addr; - grpc_error_handle error = grpc_string_to_sockaddr( - &destination_addr, host.c_str(), 0 /* port doesn't matter here */); - if (error != GRPC_ERROR_NONE) { - gpr_log(GPR_DEBUG, "Could not parse string to socket address: %s", - host.c_str()); - GRPC_ERROR_UNREF(error); + auto destination_addr = + StringToSockaddr(host, 0); // Port doesn't matter here. + if (!destination_addr.ok()) { + gpr_log(GPR_DEBUG, "Could not parse \"%s\" as socket address: %s", + host.c_str(), destination_addr.status().ToString().c_str()); return nullptr; } const XdsListenerResource::FilterChainMap::DestinationIp* best_match = @@ -972,7 +1037,7 @@ const XdsListenerResource::FilterChainData* FindFilterChainDataForDestinationIp( entry.prefix_range->prefix_len) { continue; } - if (grpc_sockaddr_match_subnet(&destination_addr, + if (grpc_sockaddr_match_subnet(&*destination_addr, &entry.prefix_range->address, entry.prefix_range->prefix_len)) { best_match = &entry; @@ -983,87 +1048,84 @@ const XdsListenerResource::FilterChainData* FindFilterChainDataForDestinationIp( host); } -absl::StatusOr<grpc_channel_args*> XdsServerConfigFetcher::ListenerWatcher:: +absl::StatusOr<ChannelArgs> XdsServerConfigFetcher::ListenerWatcher:: FilterChainMatchManager::UpdateChannelArgsForConnection( - grpc_channel_args* args, grpc_endpoint* tcp) { + const ChannelArgs& input_args, grpc_endpoint* tcp) { + ChannelArgs args = input_args; const auto* filter_chain = FindFilterChainDataForDestinationIp( filter_chain_map_.destination_ip_vector, tcp); if (filter_chain == nullptr && default_filter_chain_.has_value()) { filter_chain = &default_filter_chain_.value(); } if (filter_chain == nullptr) { - grpc_channel_args_destroy(args); return absl::UnavailableError("No matching filter chain found"); } - absl::InlinedVector<grpc_arg, 3> args_to_add; RefCountedPtr<ServerConfigSelectorProvider> server_config_selector_provider; RefCountedPtr<XdsChannelStackModifier> channel_stack_modifier; RefCountedPtr<XdsCertificateProvider> xds_certificate_provider; - // Add config selector filter - if (XdsRbacEnabled()) { - std::vector<const grpc_channel_filter*> filters; - // Iterate the list of HTTP filters in reverse since in Core, received data - // flows *up* the stack. - for (const auto& http_filter : - filter_chain->http_connection_manager.http_filters) { - // Find filter. This is guaranteed to succeed, because it's checked - // at config validation time in the XdsApi code. - const XdsHttpFilterImpl* filter_impl = - XdsHttpFilterRegistry::GetFilterForType( - http_filter.config.config_proto_type_name); - GPR_ASSERT(filter_impl != nullptr); - // Some filters like the router filter are no-op filters and do not have - // an implementation. - if (filter_impl->channel_filter() != nullptr) { - filters.push_back(filter_impl->channel_filter()); - } - } - filters.push_back(&kServerConfigSelectorFilter); - channel_stack_modifier = - MakeRefCounted<XdsChannelStackModifier>(std::move(filters)); - if (filter_chain->http_connection_manager.rds_update.has_value()) { - server_config_selector_provider = - MakeRefCounted<StaticXdsServerConfigSelectorProvider>( - filter_chain->http_connection_manager.rds_update.value(), - filter_chain->http_connection_manager.http_filters); - } else { - absl::StatusOr<XdsRouteConfigResource> initial_resource; - { - MutexLock lock(&mu_); - initial_resource = - rds_map_[filter_chain->http_connection_manager.route_config_name] - .rds_update.value(); - } - server_config_selector_provider = - MakeRefCounted<DynamicXdsServerConfigSelectorProvider>( - xds_client_, - filter_chain->http_connection_manager.route_config_name, - std::move(initial_resource), - filter_chain->http_connection_manager.http_filters); + // Iterate the list of HTTP filters in reverse since in Core, received data + // flows *up* the stack. + std::vector<const grpc_channel_filter*> filters; + const auto& http_filter_registry = + static_cast<const GrpcXdsBootstrap&>(xds_client_->bootstrap()) + .http_filter_registry(); + for (const auto& http_filter : + filter_chain->http_connection_manager.http_filters) { + // Find filter. This is guaranteed to succeed, because it's checked + // at config validation time in the XdsApi code. + const XdsHttpFilterImpl* filter_impl = + http_filter_registry.GetFilterForType( + http_filter.config.config_proto_type_name); + GPR_ASSERT(filter_impl != nullptr); + // Some filters like the router filter are no-op filters and do not have + // an implementation. + if (filter_impl->channel_filter() != nullptr) { + filters.push_back(filter_impl->channel_filter()); } - args_to_add.emplace_back(server_config_selector_provider->MakeChannelArg()); - args_to_add.emplace_back(channel_stack_modifier->MakeChannelArg()); } + // Add config selector filter. + filters.push_back(&kServerConfigSelectorFilter); + channel_stack_modifier = + MakeRefCounted<XdsChannelStackModifier>(std::move(filters)); + Match( + filter_chain->http_connection_manager.route_config, + // RDS resource name + [&](const std::string& rds_name) { + absl::StatusOr<XdsRouteConfigResource> initial_resource; + { + MutexLock lock(&mu_); + initial_resource = rds_map_[rds_name].rds_update.value(); + } + server_config_selector_provider = + MakeRefCounted<DynamicXdsServerConfigSelectorProvider>( + xds_client_->Ref(DEBUG_LOCATION, + "DynamicXdsServerConfigSelectorProvider"), + rds_name, std::move(initial_resource), + filter_chain->http_connection_manager.http_filters); + }, + // inline RouteConfig + [&](const XdsRouteConfigResource& route_config) { + server_config_selector_provider = + MakeRefCounted<StaticXdsServerConfigSelectorProvider>( + xds_client_->Ref(DEBUG_LOCATION, + "StaticXdsServerConfigSelectorProvider"), + route_config, + filter_chain->http_connection_manager.http_filters); + }); + args = args.SetObject(server_config_selector_provider) + .SetObject(channel_stack_modifier); // Add XdsCertificateProvider if credentials are xDS. - grpc_server_credentials* server_creds = - grpc_find_server_credentials_in_args(args); + auto* server_creds = args.GetObject<grpc_server_credentials>(); if (server_creds != nullptr && server_creds->type() == XdsServerCredentials::Type()) { absl::StatusOr<RefCountedPtr<XdsCertificateProvider>> result = CreateOrGetXdsCertificateProviderFromFilterChainData(filter_chain); if (!result.ok()) { - grpc_channel_args_destroy(args); return result.status(); } xds_certificate_provider = std::move(*result); GPR_ASSERT(xds_certificate_provider != nullptr); - args_to_add.emplace_back(xds_certificate_provider->MakeChannelArg()); - } - if (!args_to_add.empty()) { - grpc_channel_args* updated_args = grpc_channel_args_copy_and_add( - args, args_to_add.data(), args_to_add.size()); - grpc_channel_args_destroy(args); - args = updated_args; + args = args.SetObject(xds_certificate_provider); } return args; } @@ -1077,6 +1139,7 @@ absl::StatusOr< FilterChainMatchManager::XdsServerConfigSelector>> XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: XdsServerConfigSelector::Create( + const XdsHttpFilterRegistry& http_filter_registry, XdsRouteConfigResource rds_update, const std::vector< XdsListenerResource::HttpConnectionManager::HttpFilter>& @@ -1093,15 +1156,13 @@ XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: config_selector_route.unsupported_action = absl::get_if<XdsRouteConfigResource::Route::NonForwardingAction>( &route.action) == nullptr; - XdsRouting::GeneratePerHttpFilterConfigsResult result = - XdsRouting::GeneratePerHTTPFilterConfigs(http_filters, vhost, route, - nullptr, nullptr); - if (result.error != GRPC_ERROR_NONE) { - return grpc_error_to_absl_status(result.error); - } + auto result = XdsRouting::GeneratePerHTTPFilterConfigs( + http_filter_registry, http_filters, vhost, route, nullptr, + ChannelArgs()); + if (!result.ok()) return result.status(); std::vector<std::string> fields; - fields.reserve(result.per_filter_configs.size()); - for (const auto& p : result.per_filter_configs) { + fields.reserve(result->per_filter_configs.size()); + for (const auto& p : result->per_filter_configs) { fields.emplace_back(absl::StrCat(" \"", p.first, "\": [\n", absl::StrJoin(p.second, ",\n"), "\n ]")); @@ -1117,43 +1178,34 @@ XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: absl::StrJoin(fields, ",\n"), "\n } ]\n" "}"); - grpc_error_handle error = GRPC_ERROR_NONE; config_selector_route.method_config = - ServiceConfigImpl::Create(result.args, json.c_str(), &error); - GPR_ASSERT(error == GRPC_ERROR_NONE); + ServiceConfigImpl::Create(result->args, json.c_str()).value(); } - grpc_channel_args_destroy(result.args); } } return config_selector; } -ServerConfigSelector::CallConfig XdsServerConfigFetcher::ListenerWatcher:: - FilterChainMatchManager::XdsServerConfigSelector::GetCallConfig( - grpc_metadata_batch* metadata) { +absl::StatusOr<ServerConfigSelector::CallConfig> +XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: + XdsServerConfigSelector::GetCallConfig(grpc_metadata_batch* metadata) { CallConfig call_config; if (metadata->get_pointer(HttpPathMetadata()) == nullptr) { - call_config.error = GRPC_ERROR_CREATE_FROM_STATIC_STRING("No path found"); - return call_config; + return absl::InternalError("no path found"); } absl::string_view path = metadata->get_pointer(HttpPathMetadata())->as_string_view(); if (metadata->get_pointer(HttpAuthorityMetadata()) == nullptr) { - call_config.error = - GRPC_ERROR_CREATE_FROM_STATIC_STRING("No authority found"); - return call_config; + return absl::InternalError("no authority found"); } absl::string_view authority = metadata->get_pointer(HttpAuthorityMetadata())->as_string_view(); auto vhost_index = XdsRouting::FindVirtualHostForDomain( VirtualHostListIterator(&virtual_hosts_), authority); if (!vhost_index.has_value()) { - call_config.error = - grpc_error_set_int(GRPC_ERROR_CREATE_FROM_CPP_STRING(absl::StrCat( - "could not find VirtualHost for ", authority, - " in RouteConfiguration")), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE); - return call_config; + return absl::UnavailableError( + absl::StrCat("could not find VirtualHost for ", authority, + " in RouteConfiguration")); } auto& virtual_host = virtual_hosts_[vhost_index.value()]; auto route_index = XdsRouting::GetRouteForRequest( @@ -1162,11 +1214,7 @@ ServerConfigSelector::CallConfig XdsServerConfigFetcher::ListenerWatcher:: auto& route = virtual_host.routes[route_index.value()]; // Found the matching route if (route.unsupported_action) { - call_config.error = grpc_error_set_int( - GRPC_ERROR_CREATE_FROM_STATIC_STRING( - "Matching route has unsupported action"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE); - return call_config; + return absl::UnavailableError("matching route has unsupported action"); } if (route.method_config != nullptr) { call_config.method_configs = @@ -1175,10 +1223,7 @@ ServerConfigSelector::CallConfig XdsServerConfigFetcher::ListenerWatcher:: } return call_config; } - call_config.error = grpc_error_set_int( - GRPC_ERROR_CREATE_FROM_STATIC_STRING("No route matched"), - GRPC_ERROR_INT_GRPC_STATUS, GRPC_STATUS_UNAVAILABLE); - return call_config; + return absl::UnavailableError("no route matched"); } // @@ -1188,7 +1233,7 @@ ServerConfigSelector::CallConfig XdsServerConfigFetcher::ListenerWatcher:: XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: DynamicXdsServerConfigSelectorProvider:: DynamicXdsServerConfigSelectorProvider( - RefCountedPtr<XdsClient> xds_client, std::string resource_name, + RefCountedPtr<GrpcXdsClient> xds_client, std::string resource_name, absl::StatusOr<XdsRouteConfigResource> initial_resource, std::vector<XdsListenerResource::HttpConnectionManager::HttpFilter> http_filters) @@ -1229,7 +1274,10 @@ XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: if (!resource.ok()) { return resource.status(); } - return XdsServerConfigSelector::Create(resource.value(), http_filters_); + return XdsServerConfigSelector::Create( + static_cast<const GrpcXdsBootstrap&>(xds_client_->bootstrap()) + .http_filter_registry(), + resource.value(), http_filters_); } void XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: @@ -1250,8 +1298,10 @@ void XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: // DynamicXdsServerConfigSelectorProvider while holding a lock, but if that // ever changes, we would want to invoke the update outside the critical // region with the use of a WorkSerializer. - watcher_->OnServerConfigSelectorUpdate( - XdsServerConfigSelector::Create(*resource_, http_filters_)); + watcher_->OnServerConfigSelectorUpdate(XdsServerConfigSelector::Create( + static_cast<const GrpcXdsBootstrap&>(xds_client_->bootstrap()) + .http_filter_registry(), + *resource_, http_filters_)); } void XdsServerConfigFetcher::ListenerWatcher::FilterChainMatchManager:: @@ -1285,24 +1335,22 @@ grpc_server_config_fetcher* grpc_server_config_fetcher_xds_create( grpc_server_xds_status_notifier notifier, const grpc_channel_args* args) { grpc_core::ApplicationCallbackExecCtx callback_exec_ctx; grpc_core::ExecCtx exec_ctx; - args = grpc_core::CoreConfiguration::Get() - .channel_args_preconditioning() - .PreconditionChannelArgs(args); + grpc_core::ChannelArgs channel_args = grpc_core::CoreConfiguration::Get() + .channel_args_preconditioning() + .PreconditionChannelArgs(args); GRPC_API_TRACE( "grpc_server_config_fetcher_xds_create(notifier={on_serving_status_" "update=%p, user_data=%p}, args=%p)", 3, (notifier.on_serving_status_update, notifier.user_data, args)); - grpc_error_handle error = GRPC_ERROR_NONE; - grpc_core::RefCountedPtr<grpc_core::XdsClient> xds_client = - grpc_core::XdsClient::GetOrCreate(args, &error); - grpc_channel_args_destroy(args); - if (error != GRPC_ERROR_NONE) { + auto xds_client = grpc_core::GrpcXdsClient::GetOrCreate( + channel_args, "XdsServerConfigFetcher"); + if (!xds_client.ok()) { gpr_log(GPR_ERROR, "Failed to create xds client: %s", - grpc_error_std_string(error).c_str()); - GRPC_ERROR_UNREF(error); + xds_client.status().ToString().c_str()); return nullptr; } - if (xds_client->bootstrap() + if (static_cast<const grpc_core::GrpcXdsBootstrap&>( + (*xds_client)->bootstrap()) .server_listener_resource_name_template() .empty()) { gpr_log(GPR_ERROR, @@ -1310,5 +1358,6 @@ grpc_server_config_fetcher* grpc_server_config_fetcher_xds_create( "file."); return nullptr; } - return new grpc_core::XdsServerConfigFetcher(std::move(xds_client), notifier); + return new grpc_core::XdsServerConfigFetcher(std::move(*xds_client), + notifier); } diff --git a/grpc/src/core/ext/xds/xds_transport.h b/grpc/src/core/ext/xds/xds_transport.h new file mode 100644 index 00000000..40be0fb1 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_transport.h @@ -0,0 +1,86 @@ +// +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_TRANSPORT_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_TRANSPORT_H + +#include <grpc/support/port_platform.h> + +#include <functional> +#include <memory> +#include <string> + +#include "absl/status/status.h" +#include "absl/strings/string_view.h" + +#include "src/core/ext/xds/xds_bootstrap.h" +#include "src/core/lib/gprpp/orphanable.h" + +namespace grpc_core { + +// A factory for creating new XdsTransport instances. +class XdsTransportFactory : public InternallyRefCounted<XdsTransportFactory> { + public: + // Represents a transport for xDS communication (e.g., a gRPC channel). + class XdsTransport : public InternallyRefCounted<XdsTransport> { + public: + // Represents a bidi streaming RPC call. + class StreamingCall : public InternallyRefCounted<StreamingCall> { + public: + // An interface for handling events on a streaming call. + class EventHandler { + public: + virtual ~EventHandler() = default; + + // Called when a SendMessage() operation completes. + virtual void OnRequestSent(bool ok) = 0; + // Called when a message is received on the stream. + virtual void OnRecvMessage(absl::string_view payload) = 0; + // Called when status is received on the stream. + virtual void OnStatusReceived(absl::Status status) = 0; + }; + + // Sends a message on the stream. When the message has been sent, + // the EventHandler::OnRequestSent() method will be called. + // Only one message will be in flight at a time; subsequent + // messages will not be sent until this one is done. + virtual void SendMessage(std::string payload) = 0; + }; + + // Create a streaming call on this transport for the specified method. + // Events on the stream will be reported to event_handler. + virtual OrphanablePtr<StreamingCall> CreateStreamingCall( + const char* method, + std::unique_ptr<StreamingCall::EventHandler> event_handler) = 0; + + // Resets connection backoff for the transport. + virtual void ResetBackoff() = 0; + }; + + // Creates a new transport for the specified server. + // The on_connectivity_failure callback will be invoked whenever there is + // a connectivity failure on the transport. + // *status will be set if there is an error creating the channel, + // although the returned channel must still accept calls (which may fail). + virtual OrphanablePtr<XdsTransport> Create( + const XdsBootstrap::XdsServer& server, + std::function<void(absl::Status)> on_connectivity_failure, + absl::Status* status) = 0; +}; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_TRANSPORT_H diff --git a/grpc/src/core/ext/xds/xds_transport_grpc.cc b/grpc/src/core/ext/xds/xds_transport_grpc.cc new file mode 100644 index 00000000..1a3750a2 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_transport_grpc.cc @@ -0,0 +1,358 @@ +// +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#include <grpc/support/port_platform.h> + +#include "src/core/ext/xds/xds_transport_grpc.h" + +#include <string.h> + +#include <functional> +#include <memory> +#include <utility> + +#include "absl/strings/str_cat.h" + +#include <grpc/byte_buffer.h> +#include <grpc/byte_buffer_reader.h> +#include <grpc/grpc.h> +#include <grpc/impl/connectivity_state.h> +#include <grpc/impl/propagation_bits.h> +#include <grpc/slice.h> +#include <grpc/support/log.h> + +#include "src/core/ext/filters/client_channel/client_channel.h" +#include "src/core/ext/xds/xds_bootstrap.h" +#include "src/core/ext/xds/xds_bootstrap_grpc.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/channel/channel_fwd.h" +#include "src/core/lib/channel/channel_stack.h" +#include "src/core/lib/config/core_configuration.h" +#include "src/core/lib/gprpp/debug_location.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/gprpp/time.h" +#include "src/core/lib/iomgr/closure.h" +#include "src/core/lib/iomgr/pollset_set.h" +#include "src/core/lib/json/json.h" +#include "src/core/lib/security/credentials/channel_creds_registry.h" +#include "src/core/lib/security/credentials/credentials.h" +#include "src/core/lib/slice/slice.h" +#include "src/core/lib/slice/slice_internal.h" +#include "src/core/lib/surface/call.h" +#include "src/core/lib/surface/channel.h" +#include "src/core/lib/surface/init_internally.h" +#include "src/core/lib/surface/lame_client.h" +#include "src/core/lib/transport/connectivity_state.h" + +namespace grpc_core { + +// +// GrpcXdsTransportFactory::GrpcXdsTransport::GrpcStreamingCall +// + +GrpcXdsTransportFactory::GrpcXdsTransport::GrpcStreamingCall::GrpcStreamingCall( + RefCountedPtr<GrpcXdsTransportFactory> factory, grpc_channel* channel, + const char* method, + std::unique_ptr<StreamingCall::EventHandler> event_handler) + : factory_(std::move(factory)), event_handler_(std::move(event_handler)) { + // Create call. + call_ = grpc_channel_create_pollset_set_call( + channel, nullptr, GRPC_PROPAGATE_DEFAULTS, factory_->interested_parties(), + StaticSlice::FromStaticString(method).c_slice(), nullptr, + Timestamp::InfFuture(), nullptr); + GPR_ASSERT(call_ != nullptr); + // Init data associated with the call. + grpc_metadata_array_init(&initial_metadata_recv_); + grpc_metadata_array_init(&trailing_metadata_recv_); + // Initialize closure to be used for sending messages. + GRPC_CLOSURE_INIT(&on_request_sent_, OnRequestSent, this, nullptr); + // Start ops on the call. + grpc_call_error call_error; + grpc_op ops[3]; + memset(ops, 0, sizeof(ops)); + // Send initial metadata. No callback for this, since we don't really + // care when it finishes. + grpc_op* op = ops; + op->op = GRPC_OP_SEND_INITIAL_METADATA; + op->data.send_initial_metadata.count = 0; + op->flags = GRPC_INITIAL_METADATA_WAIT_FOR_READY | + GRPC_INITIAL_METADATA_WAIT_FOR_READY_EXPLICITLY_SET; + op->reserved = nullptr; + op++; + call_error = grpc_call_start_batch_and_execute( + call_, ops, static_cast<size_t>(op - ops), nullptr); + GPR_ASSERT(GRPC_CALL_OK == call_error); + // Start a batch with recv_initial_metadata and recv_message. + op = ops; + op->op = GRPC_OP_RECV_INITIAL_METADATA; + op->data.recv_initial_metadata.recv_initial_metadata = + &initial_metadata_recv_; + op->flags = 0; + op->reserved = nullptr; + op++; + op->op = GRPC_OP_RECV_MESSAGE; + op->data.recv_message.recv_message = &recv_message_payload_; + op->flags = 0; + op->reserved = nullptr; + op++; + Ref(DEBUG_LOCATION, "OnResponseReceived").release(); + GRPC_CLOSURE_INIT(&on_response_received_, OnResponseReceived, this, nullptr); + call_error = grpc_call_start_batch_and_execute( + call_, ops, static_cast<size_t>(op - ops), &on_response_received_); + GPR_ASSERT(GRPC_CALL_OK == call_error); + // Start a batch for recv_trailing_metadata. + op = ops; + op->op = GRPC_OP_RECV_STATUS_ON_CLIENT; + op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv_; + op->data.recv_status_on_client.status = &status_code_; + op->data.recv_status_on_client.status_details = &status_details_; + op->flags = 0; + op->reserved = nullptr; + op++; + // This callback signals the end of the call, so it relies on the initial + // ref instead of a new ref. When it's invoked, it's the initial ref that is + // unreffed. + GRPC_CLOSURE_INIT(&on_status_received_, OnStatusReceived, this, nullptr); + call_error = grpc_call_start_batch_and_execute( + call_, ops, static_cast<size_t>(op - ops), &on_status_received_); + GPR_ASSERT(GRPC_CALL_OK == call_error); +} + +GrpcXdsTransportFactory::GrpcXdsTransport::GrpcStreamingCall:: + ~GrpcStreamingCall() { + grpc_metadata_array_destroy(&initial_metadata_recv_); + grpc_metadata_array_destroy(&trailing_metadata_recv_); + grpc_byte_buffer_destroy(send_message_payload_); + grpc_byte_buffer_destroy(recv_message_payload_); + CSliceUnref(status_details_); + GPR_ASSERT(call_ != nullptr); + grpc_call_unref(call_); +} + +void GrpcXdsTransportFactory::GrpcXdsTransport::GrpcStreamingCall::Orphan() { + GPR_ASSERT(call_ != nullptr); + // If we are here because xds_client wants to cancel the call, + // OnStatusReceived() will complete the cancellation and clean up. + // Otherwise, we are here because xds_client has to orphan a failed call, + // in which case the following cancellation will be a no-op. + grpc_call_cancel_internal(call_); + // Note that the initial ref is held by OnStatusReceived(), so the + // corresponding unref happens there instead of here. +} + +void GrpcXdsTransportFactory::GrpcXdsTransport::GrpcStreamingCall::SendMessage( + std::string payload) { + // Create payload. + grpc_slice slice = grpc_slice_from_cpp_string(std::move(payload)); + send_message_payload_ = grpc_raw_byte_buffer_create(&slice, 1); + CSliceUnref(slice); + // Send the message. + grpc_op op; + memset(&op, 0, sizeof(op)); + op.op = GRPC_OP_SEND_MESSAGE; + op.data.send_message.send_message = send_message_payload_; + Ref(DEBUG_LOCATION, "OnRequestSent").release(); + grpc_call_error call_error = + grpc_call_start_batch_and_execute(call_, &op, 1, &on_request_sent_); + GPR_ASSERT(GRPC_CALL_OK == call_error); +} + +void GrpcXdsTransportFactory::GrpcXdsTransport::GrpcStreamingCall:: + OnRequestSent(void* arg, grpc_error_handle error) { + auto* self = static_cast<GrpcStreamingCall*>(arg); + // Clean up the sent message. + grpc_byte_buffer_destroy(self->send_message_payload_); + self->send_message_payload_ = nullptr; + // Invoke request handler. + self->event_handler_->OnRequestSent(error.ok()); + // Drop the ref. + self->Unref(DEBUG_LOCATION, "OnRequestSent"); +} + +void GrpcXdsTransportFactory::GrpcXdsTransport::GrpcStreamingCall:: + OnResponseReceived(void* arg, grpc_error_handle /*error*/) { + auto* self = static_cast<GrpcStreamingCall*>(arg); + // If there was no payload, then we received status before we received + // another message, so we stop reading. + if (self->recv_message_payload_ == nullptr) { + self->Unref(DEBUG_LOCATION, "OnResponseReceived"); + return; + } + // Process the response. + grpc_byte_buffer_reader bbr; + grpc_byte_buffer_reader_init(&bbr, self->recv_message_payload_); + grpc_slice response_slice = grpc_byte_buffer_reader_readall(&bbr); + grpc_byte_buffer_reader_destroy(&bbr); + grpc_byte_buffer_destroy(self->recv_message_payload_); + self->recv_message_payload_ = nullptr; + self->event_handler_->OnRecvMessage(StringViewFromSlice(response_slice)); + CSliceUnref(response_slice); + // Keep reading. + grpc_op op; + memset(&op, 0, sizeof(op)); + op.op = GRPC_OP_RECV_MESSAGE; + op.data.recv_message.recv_message = &self->recv_message_payload_; + GPR_ASSERT(self->call_ != nullptr); + // Reuses the "OnResponseReceived" ref taken in ctor. + const grpc_call_error call_error = grpc_call_start_batch_and_execute( + self->call_, &op, 1, &self->on_response_received_); + GPR_ASSERT(GRPC_CALL_OK == call_error); +} + +void GrpcXdsTransportFactory::GrpcXdsTransport::GrpcStreamingCall:: + OnStatusReceived(void* arg, grpc_error_handle /*error*/) { + auto* self = static_cast<GrpcStreamingCall*>(arg); + self->event_handler_->OnStatusReceived( + absl::Status(static_cast<absl::StatusCode>(self->status_code_), + StringViewFromSlice(self->status_details_))); + self->Unref(DEBUG_LOCATION, "OnStatusReceived"); +} + +// +// GrpcXdsTransportFactory::GrpcXdsTransport::StateWatcher +// + +class GrpcXdsTransportFactory::GrpcXdsTransport::StateWatcher + : public AsyncConnectivityStateWatcherInterface { + public: + explicit StateWatcher( + std::function<void(absl::Status)> on_connectivity_failure) + : on_connectivity_failure_(std::move(on_connectivity_failure)) {} + + private: + void OnConnectivityStateChange(grpc_connectivity_state new_state, + const absl::Status& status) override { + if (new_state == GRPC_CHANNEL_TRANSIENT_FAILURE) { + on_connectivity_failure_(absl::Status( + status.code(), + absl::StrCat("channel in TRANSIENT_FAILURE: ", status.message()))); + } + } + + std::function<void(absl::Status)> on_connectivity_failure_; +}; + +// +// GrpcXdsClient::GrpcXdsTransport +// + +namespace { + +grpc_channel* CreateXdsChannel(const ChannelArgs& args, + const GrpcXdsBootstrap::GrpcXdsServer& server) { + RefCountedPtr<grpc_channel_credentials> channel_creds = + CoreConfiguration::Get().channel_creds_registry().CreateChannelCreds( + server.channel_creds_type(), + Json::FromObject(server.channel_creds_config())); + return grpc_channel_create(server.server_uri().c_str(), channel_creds.get(), + args.ToC().get()); +} + +bool IsLameChannel(grpc_channel* channel) { + grpc_channel_element* elem = + grpc_channel_stack_last_element(grpc_channel_get_channel_stack(channel)); + return elem->filter == &LameClientFilter::kFilter; +} + +} // namespace + +GrpcXdsTransportFactory::GrpcXdsTransport::GrpcXdsTransport( + GrpcXdsTransportFactory* factory, const XdsBootstrap::XdsServer& server, + std::function<void(absl::Status)> on_connectivity_failure, + absl::Status* status) + : factory_(factory) { + channel_ = CreateXdsChannel( + factory->args_, + static_cast<const GrpcXdsBootstrap::GrpcXdsServer&>(server)); + GPR_ASSERT(channel_ != nullptr); + if (IsLameChannel(channel_)) { + *status = absl::UnavailableError("xds client has a lame channel"); + } else { + ClientChannel* client_channel = + ClientChannel::GetFromChannel(Channel::FromC(channel_)); + GPR_ASSERT(client_channel != nullptr); + watcher_ = new StateWatcher(std::move(on_connectivity_failure)); + client_channel->AddConnectivityWatcher( + GRPC_CHANNEL_IDLE, + OrphanablePtr<AsyncConnectivityStateWatcherInterface>(watcher_)); + } +} + +GrpcXdsTransportFactory::GrpcXdsTransport::~GrpcXdsTransport() { + grpc_channel_destroy_internal(channel_); +} + +void GrpcXdsTransportFactory::GrpcXdsTransport::Orphan() { + if (!IsLameChannel(channel_)) { + ClientChannel* client_channel = + ClientChannel::GetFromChannel(Channel::FromC(channel_)); + GPR_ASSERT(client_channel != nullptr); + client_channel->RemoveConnectivityWatcher(watcher_); + } + Unref(); +} + +OrphanablePtr<XdsTransportFactory::XdsTransport::StreamingCall> +GrpcXdsTransportFactory::GrpcXdsTransport::CreateStreamingCall( + const char* method, + std::unique_ptr<StreamingCall::EventHandler> event_handler) { + return MakeOrphanable<GrpcStreamingCall>( + factory_->Ref(DEBUG_LOCATION, "StreamingCall"), channel_, method, + std::move(event_handler)); +} + +void GrpcXdsTransportFactory::GrpcXdsTransport::ResetBackoff() { + grpc_channel_reset_connect_backoff(channel_); +} + +// +// GrpcXdsTransportFactory +// + +namespace { + +ChannelArgs ModifyChannelArgs(const ChannelArgs& args) { + return args.Set(GRPC_ARG_KEEPALIVE_TIME_MS, Duration::Minutes(5).millis()); +} + +} // namespace + +GrpcXdsTransportFactory::GrpcXdsTransportFactory(const ChannelArgs& args) + : args_(ModifyChannelArgs(args)), + interested_parties_(grpc_pollset_set_create()) { + // Calling grpc_init to ensure gRPC does not shut down until the XdsClient is + // destroyed. + InitInternally(); +} + +GrpcXdsTransportFactory::~GrpcXdsTransportFactory() { + grpc_pollset_set_destroy(interested_parties_); + // Calling grpc_shutdown to ensure gRPC does not shut down until the XdsClient + // is destroyed. + ShutdownInternally(); +} + +OrphanablePtr<XdsTransportFactory::XdsTransport> +GrpcXdsTransportFactory::Create( + const XdsBootstrap::XdsServer& server, + std::function<void(absl::Status)> on_connectivity_failure, + absl::Status* status) { + return MakeOrphanable<GrpcXdsTransport>( + this, server, std::move(on_connectivity_failure), status); +} + +} // namespace grpc_core diff --git a/grpc/src/core/ext/xds/xds_transport_grpc.h b/grpc/src/core/ext/xds/xds_transport_grpc.h new file mode 100644 index 00000000..f9e29fc0 --- /dev/null +++ b/grpc/src/core/ext/xds/xds_transport_grpc.h @@ -0,0 +1,135 @@ +// +// Copyright 2022 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +#ifndef GRPC_SRC_CORE_EXT_XDS_XDS_TRANSPORT_GRPC_H +#define GRPC_SRC_CORE_EXT_XDS_XDS_TRANSPORT_GRPC_H + +#include <grpc/support/port_platform.h> + +#include <functional> +#include <memory> +#include <string> + +#include "absl/status/status.h" + +#include <grpc/grpc.h> +#include <grpc/slice.h> +#include <grpc/status.h> + +#include "src/core/ext/xds/xds_bootstrap.h" +#include "src/core/ext/xds/xds_transport.h" +#include "src/core/lib/channel/channel_args.h" +#include "src/core/lib/gprpp/orphanable.h" +#include "src/core/lib/gprpp/ref_counted_ptr.h" +#include "src/core/lib/iomgr/closure.h" +#include "src/core/lib/iomgr/error.h" +#include "src/core/lib/iomgr/iomgr_fwd.h" + +namespace grpc_core { + +class GrpcXdsTransportFactory : public XdsTransportFactory { + public: + class GrpcXdsTransport; + + explicit GrpcXdsTransportFactory(const ChannelArgs& args); + ~GrpcXdsTransportFactory() override; + + void Orphan() override { Unref(); } + + OrphanablePtr<XdsTransport> Create( + const XdsBootstrap::XdsServer& server, + std::function<void(absl::Status)> on_connectivity_failure, + absl::Status* status) override; + + grpc_pollset_set* interested_parties() const { return interested_parties_; } + + private: + ChannelArgs args_; + grpc_pollset_set* interested_parties_; +}; + +class GrpcXdsTransportFactory::GrpcXdsTransport + : public XdsTransportFactory::XdsTransport { + public: + class GrpcStreamingCall; + + GrpcXdsTransport(GrpcXdsTransportFactory* factory, + const XdsBootstrap::XdsServer& server, + std::function<void(absl::Status)> on_connectivity_failure, + absl::Status* status); + ~GrpcXdsTransport() override; + + void Orphan() override; + + OrphanablePtr<StreamingCall> CreateStreamingCall( + const char* method, + std::unique_ptr<StreamingCall::EventHandler> event_handler) override; + + void ResetBackoff() override; + + private: + class StateWatcher; + + GrpcXdsTransportFactory* factory_; // Not owned. + grpc_channel* channel_; + StateWatcher* watcher_; +}; + +class GrpcXdsTransportFactory::GrpcXdsTransport::GrpcStreamingCall + : public XdsTransportFactory::XdsTransport::StreamingCall { + public: + GrpcStreamingCall(RefCountedPtr<GrpcXdsTransportFactory> factory, + grpc_channel* channel, const char* method, + std::unique_ptr<StreamingCall::EventHandler> event_handler); + ~GrpcStreamingCall() override; + + void Orphan() override; + + void SendMessage(std::string payload) override; + + private: + static void OnRequestSent(void* arg, grpc_error_handle error); + static void OnResponseReceived(void* arg, grpc_error_handle /*error*/); + static void OnStatusReceived(void* arg, grpc_error_handle /*error*/); + + RefCountedPtr<GrpcXdsTransportFactory> factory_; + + std::unique_ptr<StreamingCall::EventHandler> event_handler_; + + // Always non-NULL. + grpc_call* call_; + + // recv_initial_metadata + grpc_metadata_array initial_metadata_recv_; + + // send_message + grpc_byte_buffer* send_message_payload_ = nullptr; + grpc_closure on_request_sent_; + + // recv_message + grpc_byte_buffer* recv_message_payload_ = nullptr; + grpc_closure on_response_received_; + + // recv_trailing_metadata + grpc_metadata_array trailing_metadata_recv_; + grpc_status_code status_code_; + grpc_slice status_details_; + grpc_closure on_status_received_; +}; + +} // namespace grpc_core + +#endif // GRPC_SRC_CORE_EXT_XDS_XDS_TRANSPORT_GRPC_H |