aboutsummaryrefslogtreecommitdiff
path: root/icing/store
diff options
context:
space:
mode:
Diffstat (limited to 'icing/store')
-rw-r--r--icing/store/document-store.cc11
-rw-r--r--icing/store/document-store.h4
-rw-r--r--icing/store/document-store_test.cc4
-rw-r--r--icing/store/key-mapper_benchmark.cc9
-rw-r--r--icing/store/key-mapper_test.cc22
-rw-r--r--icing/store/persistent-hash-map-key-mapper.h54
6 files changed, 53 insertions, 51 deletions
diff --git a/icing/store/document-store.cc b/icing/store/document-store.cc
index 9e79790..2a7e108 100644
--- a/icing/store/document-store.cc
+++ b/icing/store/document-store.cc
@@ -1873,8 +1873,7 @@ libtextclassifier3::Status DocumentStore::SetUsageScores(
libtextclassifier3::StatusOr<
google::protobuf::RepeatedPtrField<DocumentDebugInfoProto::CorpusInfo>>
DocumentStore::CollectCorpusInfo() const {
- google::protobuf::RepeatedPtrField<DocumentDebugInfoProto::CorpusInfo>
- corpus_info;
+ google::protobuf::RepeatedPtrField<DocumentDebugInfoProto::CorpusInfo> corpus_info;
libtextclassifier3::StatusOr<const SchemaProto*> schema_proto_or =
schema_store_->GetSchema();
if (!schema_proto_or.ok()) {
@@ -1919,10 +1918,10 @@ DocumentStore::GetDebugInfo(int verbosity) const {
ICING_ASSIGN_OR_RETURN(Crc32 crc, ComputeChecksum());
debug_info.set_crc(crc.Get());
if (verbosity > 0) {
- ICING_ASSIGN_OR_RETURN(google::protobuf::RepeatedPtrField<
- DocumentDebugInfoProto::CorpusInfo>
- corpus_info,
- CollectCorpusInfo());
+ ICING_ASSIGN_OR_RETURN(
+ google::protobuf::RepeatedPtrField<DocumentDebugInfoProto::CorpusInfo>
+ corpus_info,
+ CollectCorpusInfo());
*debug_info.mutable_corpus_info() = std::move(corpus_info);
}
return debug_info;
diff --git a/icing/store/document-store.h b/icing/store/document-store.h
index bda351d..3e02636 100644
--- a/icing/store/document-store.h
+++ b/icing/store/document-store.h
@@ -730,8 +730,8 @@ class DocumentStore {
// Returns:
// - on success, a RepeatedPtrField for CorpusInfo collected.
// - OUT_OF_RANGE, this should never happen.
- libtextclassifier3::StatusOr<google::protobuf::RepeatedPtrField<
- DocumentDebugInfoProto::CorpusInfo>>
+ libtextclassifier3::StatusOr<
+ google::protobuf::RepeatedPtrField<DocumentDebugInfoProto::CorpusInfo>>
CollectCorpusInfo() const;
};
diff --git a/icing/store/document-store_test.cc b/icing/store/document-store_test.cc
index 7cf951a..a115e11 100644
--- a/icing/store/document-store_test.cc
+++ b/icing/store/document-store_test.cc
@@ -88,7 +88,9 @@ const NamespaceStorageInfoProto& GetNamespaceStorageInfo(
// Didn't find our namespace, fail the test.
EXPECT_TRUE(false) << "Failed to find namespace '" << name_space
<< "' in DocumentStorageInfoProto.";
- return std::move(NamespaceStorageInfoProto());
+ static const auto& default_namespace_storage_info =
+ *new NamespaceStorageInfoProto();
+ return default_namespace_storage_info;
}
UsageReport CreateUsageReport(std::string name_space, std::string uri,
diff --git a/icing/store/key-mapper_benchmark.cc b/icing/store/key-mapper_benchmark.cc
index b649bc7..1ce54c7 100644
--- a/icing/store/key-mapper_benchmark.cc
+++ b/icing/store/key-mapper_benchmark.cc
@@ -35,6 +35,7 @@ namespace lib {
namespace {
using ::testing::Eq;
+using ::testing::IsTrue;
using ::testing::Not;
class KeyMapperBenchmark {
@@ -78,8 +79,10 @@ class KeyMapperBenchmark {
template <>
libtextclassifier3::StatusOr<std::unique_ptr<KeyMapper<int>>>
CreateKeyMapper<PersistentHashMapKeyMapper<int>>(int max_num_entries) {
+ std::string working_path =
+ absl_ports::StrCat(base_dir, "/", "key_mapper_dir");
return PersistentHashMapKeyMapper<int>::Create(
- filesystem, base_dir, max_num_entries,
+ filesystem, std::move(working_path), max_num_entries,
/*average_kv_byte_size=*/kKeyLength + 1 + sizeof(int),
/*max_load_factor_percent=*/100);
}
@@ -109,6 +112,7 @@ void BM_PutMany(benchmark::State& state) {
state.PauseTiming();
benchmark.filesystem.DeleteDirectoryRecursively(benchmark.base_dir.c_str());
DestructibleDirectory ddir(&benchmark.filesystem, benchmark.base_dir);
+ ASSERT_THAT(ddir.is_valid(), IsTrue());
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<KeyMapper<int>> key_mapper,
benchmark.CreateKeyMapper<KeyMapperType>(num_keys));
@@ -166,6 +170,7 @@ void BM_Put(benchmark::State& state) {
KeyMapperBenchmark benchmark;
benchmark.filesystem.DeleteDirectoryRecursively(benchmark.base_dir.c_str());
DestructibleDirectory ddir(&benchmark.filesystem, benchmark.base_dir);
+ ASSERT_THAT(ddir.is_valid(), IsTrue());
// The overhead of state.PauseTiming is too large and affects the benchmark
// result a lot, so pre-generate enough kvps to avoid calling too many times
@@ -206,6 +211,7 @@ void BM_Get(benchmark::State& state) {
KeyMapperBenchmark benchmark;
benchmark.filesystem.DeleteDirectoryRecursively(benchmark.base_dir.c_str());
DestructibleDirectory ddir(&benchmark.filesystem, benchmark.base_dir);
+ ASSERT_THAT(ddir.is_valid(), IsTrue());
// Create a key mapper with num_keys entries.
ICING_ASSERT_OK_AND_ASSIGN(
@@ -260,6 +266,7 @@ void BM_Iterator(benchmark::State& state) {
KeyMapperBenchmark benchmark;
benchmark.filesystem.DeleteDirectoryRecursively(benchmark.base_dir.c_str());
DestructibleDirectory ddir(&benchmark.filesystem, benchmark.base_dir);
+ ASSERT_THAT(ddir.is_valid(), IsTrue());
// Create a key mapper with num_keys entries.
ICING_ASSERT_OK_AND_ASSIGN(
diff --git a/icing/store/key-mapper_test.cc b/icing/store/key-mapper_test.cc
index 682888d..1367c2d 100644
--- a/icing/store/key-mapper_test.cc
+++ b/icing/store/key-mapper_test.cc
@@ -32,6 +32,7 @@
#include "icing/testing/tmp-directory.h"
using ::testing::IsEmpty;
+using ::testing::IsTrue;
using ::testing::Pair;
using ::testing::UnorderedElementsAre;
@@ -47,7 +48,13 @@ class KeyMapperTest : public ::testing::Test {
protected:
using KeyMapperType = T;
- void SetUp() override { base_dir_ = GetTestTempDir() + "/key_mapper"; }
+ void SetUp() override {
+ base_dir_ = GetTestTempDir() + "/icing";
+ ASSERT_THAT(filesystem_.CreateDirectoryRecursively(base_dir_.c_str()),
+ IsTrue());
+
+ working_dir_ = base_dir_ + "/key_mapper";
+ }
void TearDown() override {
filesystem_.DeleteDirectoryRecursively(base_dir_.c_str());
@@ -63,17 +70,18 @@ class KeyMapperTest : public ::testing::Test {
libtextclassifier3::StatusOr<std::unique_ptr<KeyMapper<DocumentId>>>
CreateKeyMapper<DynamicTrieKeyMapper<DocumentId>>() {
return DynamicTrieKeyMapper<DocumentId>::Create(
- filesystem_, base_dir_, kMaxDynamicTrieKeyMapperSize);
+ filesystem_, working_dir_, kMaxDynamicTrieKeyMapperSize);
}
template <>
libtextclassifier3::StatusOr<std::unique_ptr<KeyMapper<DocumentId>>>
CreateKeyMapper<PersistentHashMapKeyMapper<DocumentId>>() {
return PersistentHashMapKeyMapper<DocumentId>::Create(filesystem_,
- base_dir_);
+ working_dir_);
}
std::string base_dir_;
+ std::string working_dir_;
Filesystem filesystem_;
};
@@ -175,15 +183,15 @@ TYPED_TEST(KeyMapperTest, CanUseAcrossMultipleInstances) {
TYPED_TEST(KeyMapperTest, CanDeleteAndRestartKeyMapping) {
// Can delete even if there's nothing there
- ICING_EXPECT_OK(
- TestFixture::KeyMapperType::Delete(this->filesystem_, this->base_dir_));
+ ICING_EXPECT_OK(TestFixture::KeyMapperType::Delete(this->filesystem_,
+ this->working_dir_));
ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<KeyMapper<DocumentId>> key_mapper,
this->template CreateKeyMapper<TypeParam>());
ICING_EXPECT_OK(key_mapper->Put("default-google.com", 100));
ICING_EXPECT_OK(key_mapper->PersistToDisk());
- ICING_EXPECT_OK(
- TestFixture::KeyMapperType::Delete(this->filesystem_, this->base_dir_));
+ ICING_EXPECT_OK(TestFixture::KeyMapperType::Delete(this->filesystem_,
+ this->working_dir_));
key_mapper.reset();
ICING_ASSERT_OK_AND_ASSIGN(key_mapper,
diff --git a/icing/store/persistent-hash-map-key-mapper.h b/icing/store/persistent-hash-map-key-mapper.h
index a13ec11..5f83e6f 100644
--- a/icing/store/persistent-hash-map-key-mapper.h
+++ b/icing/store/persistent-hash-map-key-mapper.h
@@ -43,11 +43,13 @@ class PersistentHashMapKeyMapper : public KeyMapper<T, Formatter> {
// Returns any encountered IO errors.
//
// filesystem: Object to make system level calls
- // base_dir : Base directory used to save all the files required to persist
- // PersistentHashMapKeyMapper. If this base_dir was previously used
- // to create a PersistentHashMapKeyMapper, then this existing data
- // would be loaded. Otherwise, an empty PersistentHashMapKeyMapper
- // would be created.
+ // working_path: Working directory used to save all the files required to
+ // persist PersistentHashMapKeyMapper. If this working_path was
+ // previously used to create a PersistentHashMapKeyMapper, then
+ // this existing data would be loaded. Otherwise, an empty
+ // PersistentHashMapKeyMapper would be created. See
+ // PersistentStorage for more details about the concept of
+ // working_path.
// max_num_entries: max # of kvps. It will be used to compute 3 storages size.
// average_kv_byte_size: average byte size of a single key + serialized value.
// It will be used to compute kv_storage size.
@@ -60,24 +62,25 @@ class PersistentHashMapKeyMapper : public KeyMapper<T, Formatter> {
// considered valid.
static libtextclassifier3::StatusOr<
std::unique_ptr<PersistentHashMapKeyMapper<T, Formatter>>>
- Create(const Filesystem& filesystem, std::string_view base_dir,
+ Create(const Filesystem& filesystem, std::string working_path,
int32_t max_num_entries = PersistentHashMap::Entry::kMaxNumEntries,
int32_t average_kv_byte_size =
PersistentHashMap::Options::kDefaultAverageKVByteSize,
int32_t max_load_factor_percent =
PersistentHashMap::Options::kDefaultMaxLoadFactorPercent);
- // Deletes all the files associated with the PersistentHashMapKeyMapper.
+ // Deletes working_path (and all the files under it recursively) associated
+ // with the PersistentHashMapKeyMapper.
//
- // base_dir : Base directory used to save all the files required to persist
- // PersistentHashMapKeyMapper. Should be the same as passed into
- // Create().
+ // working_path: Working directory used to save all the files required to
+ // persist PersistentHashMapKeyMapper. Should be the same as
+ // passed into Create().
//
// Returns:
// OK on success
// INTERNAL_ERROR on I/O error
static libtextclassifier3::Status Delete(const Filesystem& filesystem,
- std::string_view base_dir);
+ const std::string& working_path);
~PersistentHashMapKeyMapper() override = default;
@@ -122,7 +125,7 @@ class PersistentHashMapKeyMapper : public KeyMapper<T, Formatter> {
}
libtextclassifier3::StatusOr<Crc32> ComputeChecksum() override {
- return persistent_hash_map_->ComputeChecksum();
+ return persistent_hash_map_->UpdateChecksums();
}
private:
@@ -147,8 +150,6 @@ class PersistentHashMapKeyMapper : public KeyMapper<T, Formatter> {
PersistentHashMap::Iterator itr_;
};
- static constexpr std::string_view kKeyMapperDir = "key_mapper_dir";
-
// Use PersistentHashMapKeyMapper::Create() to instantiate.
explicit PersistentHashMapKeyMapper(
std::unique_ptr<PersistentHashMap> persistent_hash_map)
@@ -164,21 +165,13 @@ template <typename T, typename Formatter>
/* static */ libtextclassifier3::StatusOr<
std::unique_ptr<PersistentHashMapKeyMapper<T, Formatter>>>
PersistentHashMapKeyMapper<T, Formatter>::Create(
- const Filesystem& filesystem, std::string_view base_dir,
+ const Filesystem& filesystem, std::string working_path,
int32_t max_num_entries, int32_t average_kv_byte_size,
int32_t max_load_factor_percent) {
- const std::string key_mapper_dir =
- absl_ports::StrCat(base_dir, "/", kKeyMapperDir);
- if (!filesystem.CreateDirectoryRecursively(key_mapper_dir.c_str())) {
- return absl_ports::InternalError(absl_ports::StrCat(
- "Failed to create PersistentHashMapKeyMapper directory: ",
- key_mapper_dir));
- }
-
ICING_ASSIGN_OR_RETURN(
std::unique_ptr<PersistentHashMap> persistent_hash_map,
PersistentHashMap::Create(
- filesystem, key_mapper_dir,
+ filesystem, std::move(working_path),
PersistentHashMap::Options(
/*value_type_size_in=*/sizeof(T),
/*max_num_entries_in=*/max_num_entries,
@@ -191,16 +184,9 @@ PersistentHashMapKeyMapper<T, Formatter>::Create(
template <typename T, typename Formatter>
/* static */ libtextclassifier3::Status
-PersistentHashMapKeyMapper<T, Formatter>::Delete(const Filesystem& filesystem,
- std::string_view base_dir) {
- const std::string key_mapper_dir =
- absl_ports::StrCat(base_dir, "/", kKeyMapperDir);
- if (!filesystem.DeleteDirectoryRecursively(key_mapper_dir.c_str())) {
- return absl_ports::InternalError(absl_ports::StrCat(
- "Failed to delete PersistentHashMapKeyMapper directory: ",
- key_mapper_dir));
- }
- return libtextclassifier3::Status::OK;
+PersistentHashMapKeyMapper<T, Formatter>::Delete(
+ const Filesystem& filesystem, const std::string& working_path) {
+ return PersistentHashMap::Discard(filesystem, working_path);
}
} // namespace lib
generated by cgit v1.2.3 (git 2.25.1) at 2024-07-18 06:45:02 +0000