From 396ee7878c98be2ad9de7120ff6e7e3745530029 Mon Sep 17 00:00:00 2001 From: "roubert@google.com" Date: Thu, 22 May 2014 11:54:14 +0000 Subject: Rename class MetadataLoader to class OndemandSupplier. With this class now being an implementation of the Supplier interface (r235) this new name better reflects that fact and its relationship to PreloadSupplier (r236) and corresponds to the class comment: An implementation of the Supplier interface that owns a Retriever object, through which it loads address metadata as needed, creating Rule objects and caching these. R=rouslan@chromium.org BUG= Review URL: https://codereview.appspot.com/96540048 git-svn-id: http://libaddressinput.googlecode.com/svn/trunk@239 38ededc0-08b8-5190-f2ac-b31f878777ad --- cpp/src/address_validator.cc | 4 +- cpp/src/metadata_loader.cc | 177 ----------------------------------------- cpp/src/metadata_loader.h | 96 ---------------------- cpp/src/metadata_query_task.cc | 2 +- cpp/src/ondemand_supplier.cc | 177 +++++++++++++++++++++++++++++++++++++++++ cpp/src/ondemand_supplier.h | 96 ++++++++++++++++++++++ cpp/src/validation_task.h | 2 +- 7 files changed, 277 insertions(+), 277 deletions(-) delete mode 100644 cpp/src/metadata_loader.cc delete mode 100644 cpp/src/metadata_loader.h create mode 100644 cpp/src/ondemand_supplier.cc create mode 100644 cpp/src/ondemand_supplier.h (limited to 'cpp/src') diff --git a/cpp/src/address_validator.cc b/cpp/src/address_validator.cc index 39ff651..1092f8c 100644 --- a/cpp/src/address_validator.cc +++ b/cpp/src/address_validator.cc @@ -23,8 +23,8 @@ #include #include -#include "metadata_loader.h" #include "metadata_query_task.h" +#include "ondemand_supplier.h" #include "retriever.h" #include "rule.h" #include "validation_task.h" @@ -35,7 +35,7 @@ namespace addressinput { AddressValidator::AddressValidator(const std::string& validation_data_url, const Downloader* downloader, Storage* storage) - : own_supplier_(new MetadataLoader( + : own_supplier_(new OndemandSupplier( new Retriever(validation_data_url, downloader, storage))), supplier_(own_supplier_.get()) { assert(supplier_ != NULL); diff --git a/cpp/src/metadata_loader.cc b/cpp/src/metadata_loader.cc deleted file mode 100644 index a86df94..0000000 --- a/cpp/src/metadata_loader.cc +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (C) 2014 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "metadata_loader.h" - -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -#include "lookup_key.h" -#include "region_data_constants.h" -#include "retriever.h" -#include "rule.h" - -namespace i18n { -namespace addressinput { - -MetadataLoader::MetadataLoader(const Retriever* retriever) - : retriever_(retriever) { - assert(retriever_ != NULL); -} - -MetadataLoader::~MetadataLoader() { - for (std::map::const_iterator - it = rule_cache_.begin(); it != rule_cache_.end(); ++it) { - delete it->second; - } -} - -void MetadataLoader::Supply(const LookupKey& lookup_key, - const Callback& supplied) { - RuleHierarchy* hierarchy = - new RuleHierarchy(lookup_key, &rule_cache_, supplied); - - if (RegionDataConstants::IsSupported(lookup_key.GetRegionCode())) { - size_t max_depth = std::min( - lookup_key.GetDepth(), - RegionDataConstants::GetMaxLookupKeyDepth(lookup_key.GetRegionCode())); - - for (size_t depth = 0; depth <= max_depth; ++depth) { - const std::string& key = lookup_key.ToKeyString(depth); - std::map::const_iterator it = - rule_cache_.find(key); - if (it != rule_cache_.end()) { - hierarchy->rule_[depth] = it->second; - } else { - hierarchy->Queue(key); // If not in the cache, it needs to be loaded. - } - } - } - - hierarchy->Retrieve(*retriever_); -} - -MetadataLoader::RuleHierarchy::RuleHierarchy( - const LookupKey& lookup_key, - std::map* rules, - const Callback& supplied) - : lookup_key_(lookup_key), - rule_cache_(rules), - supplied_(supplied), - retrieved_(BuildCallback(this, &MetadataLoader::RuleHierarchy::Load)), - success_(true) { - assert(rule_cache_ != NULL); - assert(retrieved_ != NULL); -} - -MetadataLoader::RuleHierarchy::~RuleHierarchy() { -} - -void MetadataLoader::RuleHierarchy::Queue(const std::string& key) { - assert(pending_.find(key) == pending_.end()); - pending_.insert(key); -} - -void MetadataLoader::RuleHierarchy::Retrieve(const Retriever& retriever) { - if (pending_.empty()) { - Loaded(); - } else { - // When the final pending rule has been retrieved, the retrieved_ callback, - // implemented by Load(), will finish by calling Loaded(), which will finish - // by delete'ing this RuleHierarchy object. So after the final call to - // retriever.Retrieve() no attributes of this object can be accessed (as the - // object then no longer will exist, if the final callback has finished by - // then), and the condition statement of the loop must therefore not use the - // otherwise obvious it != pending_.end() but instead test a local variable - // that isn't affected by the object being deleted. - bool done = false; - for (std::set::const_iterator - it = pending_.begin(); !done; ) { - const std::string& key = *it++; - done = it == pending_.end(); - retriever.Retrieve(key, *retrieved_); - } - } -} - -void MetadataLoader::RuleHierarchy::Load(bool success, - const std::string& key, - const std::string& data) { - // Sanity check: This key should be present in the set of pending requests. - size_t status = pending_.erase(key); - assert(status == 1); // There will always be one item erased from the set. - (void)status; // Prevent unused variable if assert() is optimized away. - - size_t depth = std::count(key.begin(), key.end(), '/') - 1; - assert(depth < arraysize(LookupKey::kHierarchy)); - AddressField field = LookupKey::kHierarchy[depth]; - - if (success) { - // The address metadata server will return the empty JSON "{}" when it - // successfully performed a lookup, but didn't find any data for that key. - if (data != "{}") { - Rule* rule = new Rule; - if (field == COUNTRY) { - // All rules on the COUNTRY level inherit from the default rule. - rule->CopyFrom(Rule::GetDefault()); - } - if (rule->ParseSerializedRule(data)) { - // Try inserting the Rule object into the rule_cache_ map, or else find - // the already existing Rule object with the same ID already in the map. - // It is possible that a key was queued even though the corresponding - // Rule object is already in the cache, as the data server is free to do - // advanced normalization and aliasing so that the ID of the data - // returned is different from the key requested. (It would be possible - // to cache such aliases, to increase performance in the case where a - // certain alias is requested repeatedly, but such a cache would then - // have to be kept to some limited size to not grow indefinitely with - // every possible permutation of a name recognized by the data server.) - std::pair::iterator, bool> result = - rule_cache_->insert(std::make_pair(rule->GetId(), rule)); - if (!result.second) { // There was already an entry with this ID. - delete rule; - } - rule_[depth] = result.first->second; // Pointer to object in the map. - } else { - delete rule; - success_ = false; - } - } - } else { - success_ = false; - } - - if (pending_.empty()) { - Loaded(); - } -} - -void MetadataLoader::RuleHierarchy::Loaded() { - supplied_(success_, lookup_key_, *this); - delete this; -} - -} // namespace addressinput -} // namespace i18n diff --git a/cpp/src/metadata_loader.h b/cpp/src/metadata_loader.h deleted file mode 100644 index e6208a8..0000000 --- a/cpp/src/metadata_loader.h +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright (C) 2014 Google Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#ifndef I18N_ADDRESSINPUT_METADATA_LOADER_H_ -#define I18N_ADDRESSINPUT_METADATA_LOADER_H_ - -#include -#include -#include -#include - -#include -#include -#include - -#include "retriever.h" - -namespace i18n { -namespace addressinput { - -class LookupKey; -class Rule; - -// An implementation of the Supplier interface that owns a Retriever object, -// through which it loads address metadata as needed, creating Rule objects and -// caching these. -// -// When using a MetadataLoader, address validation will benefit from address -// metadata server synonym resolution, because the server will be contacted for -// every new LookupKey (ie. every LookupKey that isn't on canonical form and -// isn't already cached). -// -// The maximum size of this cache is naturally limited to the amount of data -// available from the data server. (Currently this is less than 12,000 items of -// in total less than 2 MB of JSON data.) -class MetadataLoader : public Supplier { - public: - // Takes ownership of |retriever|. - MetadataLoader(const Retriever* retriever); - virtual ~MetadataLoader(); - - // Loads the metadata needed for |lookup_key|, then calls |supplied|. - virtual void Supply(const LookupKey& lookup_key, const Callback& supplied); - - // A RuleHierarchy object encapsulates the set of Rule objects corresponding - // to a LookupKey, together with methods for retrieving and parsing data as - // necessary from a Retriever object. - class RuleHierarchy : public Supplier::RuleHierarchy { - public: - RuleHierarchy(const LookupKey& lookup_key, - std::map* rules, - const Callback& supplied); - virtual ~RuleHierarchy(); - - // Adds lookup key string |key| to the queue of data to be retrieved. - void Queue(const std::string& key); - - // Retrieves and parses data for all queued keys, then calls |supplied_|. - void Retrieve(const Retriever& retriever); - - private: - void Load(bool success, const std::string& key, const std::string& data); - void Loaded(); - - std::set pending_; - const LookupKey& lookup_key_; - std::map* const rule_cache_; - const Callback& supplied_; - const scoped_ptr retrieved_; - bool success_; - - DISALLOW_COPY_AND_ASSIGN(RuleHierarchy); - }; - - private: - const scoped_ptr retriever_; - std::map rule_cache_; - - DISALLOW_COPY_AND_ASSIGN(MetadataLoader); -}; - -} // namespace addressinput -} // namespace i18n - -#endif // I18N_ADDRESSINPUT_METADATA_LOADER_H_ diff --git a/cpp/src/metadata_query_task.cc b/cpp/src/metadata_query_task.cc index ed5483c..13b827c 100644 --- a/cpp/src/metadata_query_task.cc +++ b/cpp/src/metadata_query_task.cc @@ -23,7 +23,7 @@ #include #include "lookup_key.h" -#include "metadata_loader.h" +#include "ondemand_supplier.h" namespace i18n { namespace addressinput { diff --git a/cpp/src/ondemand_supplier.cc b/cpp/src/ondemand_supplier.cc new file mode 100644 index 0000000..38b4e7c --- /dev/null +++ b/cpp/src/ondemand_supplier.cc @@ -0,0 +1,177 @@ +// Copyright (C) 2014 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "ondemand_supplier.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "lookup_key.h" +#include "region_data_constants.h" +#include "retriever.h" +#include "rule.h" + +namespace i18n { +namespace addressinput { + +OndemandSupplier::OndemandSupplier(const Retriever* retriever) + : retriever_(retriever) { + assert(retriever_ != NULL); +} + +OndemandSupplier::~OndemandSupplier() { + for (std::map::const_iterator + it = rule_cache_.begin(); it != rule_cache_.end(); ++it) { + delete it->second; + } +} + +void OndemandSupplier::Supply(const LookupKey& lookup_key, + const Callback& supplied) { + RuleHierarchy* hierarchy = + new RuleHierarchy(lookup_key, &rule_cache_, supplied); + + if (RegionDataConstants::IsSupported(lookup_key.GetRegionCode())) { + size_t max_depth = std::min( + lookup_key.GetDepth(), + RegionDataConstants::GetMaxLookupKeyDepth(lookup_key.GetRegionCode())); + + for (size_t depth = 0; depth <= max_depth; ++depth) { + const std::string& key = lookup_key.ToKeyString(depth); + std::map::const_iterator it = + rule_cache_.find(key); + if (it != rule_cache_.end()) { + hierarchy->rule_[depth] = it->second; + } else { + hierarchy->Queue(key); // If not in the cache, it needs to be loaded. + } + } + } + + hierarchy->Retrieve(*retriever_); +} + +OndemandSupplier::RuleHierarchy::RuleHierarchy( + const LookupKey& lookup_key, + std::map* rules, + const Callback& supplied) + : lookup_key_(lookup_key), + rule_cache_(rules), + supplied_(supplied), + retrieved_(BuildCallback(this, &OndemandSupplier::RuleHierarchy::Load)), + success_(true) { + assert(rule_cache_ != NULL); + assert(retrieved_ != NULL); +} + +OndemandSupplier::RuleHierarchy::~RuleHierarchy() { +} + +void OndemandSupplier::RuleHierarchy::Queue(const std::string& key) { + assert(pending_.find(key) == pending_.end()); + pending_.insert(key); +} + +void OndemandSupplier::RuleHierarchy::Retrieve(const Retriever& retriever) { + if (pending_.empty()) { + Loaded(); + } else { + // When the final pending rule has been retrieved, the retrieved_ callback, + // implemented by Load(), will finish by calling Loaded(), which will finish + // by delete'ing this RuleHierarchy object. So after the final call to + // retriever.Retrieve() no attributes of this object can be accessed (as the + // object then no longer will exist, if the final callback has finished by + // then), and the condition statement of the loop must therefore not use the + // otherwise obvious it != pending_.end() but instead test a local variable + // that isn't affected by the object being deleted. + bool done = false; + for (std::set::const_iterator + it = pending_.begin(); !done; ) { + const std::string& key = *it++; + done = it == pending_.end(); + retriever.Retrieve(key, *retrieved_); + } + } +} + +void OndemandSupplier::RuleHierarchy::Load(bool success, + const std::string& key, + const std::string& data) { + // Sanity check: This key should be present in the set of pending requests. + size_t status = pending_.erase(key); + assert(status == 1); // There will always be one item erased from the set. + (void)status; // Prevent unused variable if assert() is optimized away. + + size_t depth = std::count(key.begin(), key.end(), '/') - 1; + assert(depth < arraysize(LookupKey::kHierarchy)); + AddressField field = LookupKey::kHierarchy[depth]; + + if (success) { + // The address metadata server will return the empty JSON "{}" when it + // successfully performed a lookup, but didn't find any data for that key. + if (data != "{}") { + Rule* rule = new Rule; + if (field == COUNTRY) { + // All rules on the COUNTRY level inherit from the default rule. + rule->CopyFrom(Rule::GetDefault()); + } + if (rule->ParseSerializedRule(data)) { + // Try inserting the Rule object into the rule_cache_ map, or else find + // the already existing Rule object with the same ID already in the map. + // It is possible that a key was queued even though the corresponding + // Rule object is already in the cache, as the data server is free to do + // advanced normalization and aliasing so that the ID of the data + // returned is different from the key requested. (It would be possible + // to cache such aliases, to increase performance in the case where a + // certain alias is requested repeatedly, but such a cache would then + // have to be kept to some limited size to not grow indefinitely with + // every possible permutation of a name recognized by the data server.) + std::pair::iterator, bool> result = + rule_cache_->insert(std::make_pair(rule->GetId(), rule)); + if (!result.second) { // There was already an entry with this ID. + delete rule; + } + rule_[depth] = result.first->second; // Pointer to object in the map. + } else { + delete rule; + success_ = false; + } + } + } else { + success_ = false; + } + + if (pending_.empty()) { + Loaded(); + } +} + +void OndemandSupplier::RuleHierarchy::Loaded() { + supplied_(success_, lookup_key_, *this); + delete this; +} + +} // namespace addressinput +} // namespace i18n diff --git a/cpp/src/ondemand_supplier.h b/cpp/src/ondemand_supplier.h new file mode 100644 index 0000000..e8e3a7b --- /dev/null +++ b/cpp/src/ondemand_supplier.h @@ -0,0 +1,96 @@ +// Copyright (C) 2014 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef I18N_ADDRESSINPUT_ONDEMAND_SUPPLIER_H_ +#define I18N_ADDRESSINPUT_ONDEMAND_SUPPLIER_H_ + +#include +#include +#include +#include + +#include +#include +#include + +#include "retriever.h" + +namespace i18n { +namespace addressinput { + +class LookupKey; +class Rule; + +// An implementation of the Supplier interface that owns a Retriever object, +// through which it loads address metadata as needed, creating Rule objects and +// caching these. +// +// When using an OndemandSupplier, address validation will benefit from address +// metadata server synonym resolution, because the server will be contacted for +// every new LookupKey (ie. every LookupKey that isn't on canonical form and +// isn't already cached). +// +// The maximum size of this cache is naturally limited to the amount of data +// available from the data server. (Currently this is less than 12,000 items of +// in total less than 2 MB of JSON data.) +class OndemandSupplier : public Supplier { + public: + // Takes ownership of |retriever|. + OndemandSupplier(const Retriever* retriever); + virtual ~OndemandSupplier(); + + // Loads the metadata needed for |lookup_key|, then calls |supplied|. + virtual void Supply(const LookupKey& lookup_key, const Callback& supplied); + + // A RuleHierarchy object encapsulates the set of Rule objects corresponding + // to a LookupKey, together with methods for retrieving and parsing data as + // necessary from a Retriever object. + class RuleHierarchy : public Supplier::RuleHierarchy { + public: + RuleHierarchy(const LookupKey& lookup_key, + std::map* rules, + const Callback& supplied); + virtual ~RuleHierarchy(); + + // Adds lookup key string |key| to the queue of data to be retrieved. + void Queue(const std::string& key); + + // Retrieves and parses data for all queued keys, then calls |supplied_|. + void Retrieve(const Retriever& retriever); + + private: + void Load(bool success, const std::string& key, const std::string& data); + void Loaded(); + + std::set pending_; + const LookupKey& lookup_key_; + std::map* const rule_cache_; + const Callback& supplied_; + const scoped_ptr retrieved_; + bool success_; + + DISALLOW_COPY_AND_ASSIGN(RuleHierarchy); + }; + + private: + const scoped_ptr retriever_; + std::map rule_cache_; + + DISALLOW_COPY_AND_ASSIGN(OndemandSupplier); +}; + +} // namespace addressinput +} // namespace i18n + +#endif // I18N_ADDRESSINPUT_ONDEMAND_SUPPLIER_H_ diff --git a/cpp/src/validation_task.h b/cpp/src/validation_task.h index 3bbc643..da0ce4b 100644 --- a/cpp/src/validation_task.h +++ b/cpp/src/validation_task.h @@ -44,7 +44,7 @@ class ValidationTask { ~ValidationTask(); - // Calls metadata->Load(), with Validate() as callback. + // Calls supplier->Load(), with Validate() as callback. void Run(Supplier* supplier) const; private: -- cgit v1.2.3