diff options
112 files changed, 10970 insertions, 1994 deletions
@@ -54,6 +54,8 @@ cc_defaults { "-funsigned-char", "-fvisibility=hidden", + + "-Bsymbolic", ], apex_available: ["com.android.appsearch"], } diff --git a/build.gradle b/build.gradle index a3aa34d..d0d1a39 100644 --- a/build.gradle +++ b/build.gradle @@ -14,66 +14,42 @@ * limitations under the License. */ -import androidx.build.SupportConfig +import androidx.build.SdkHelperKt plugins { - id('AndroidXPlugin') - id('com.android.library') - id('com.google.protobuf') + id("AndroidXPlugin") + id("java-library") + id("com.google.protobuf") } -android { - buildToolsVersion SupportConfig.buildToolsVersion(project) - compileSdkVersion SupportConfig.COMPILE_SDK_VERSION - defaultConfig { - minSdkVersion SupportConfig.DEFAULT_MIN_SDK_VERSION - targetSdkVersion SupportConfig.TARGET_SDK_VERSION - testInstrumentationRunner SupportConfig.INSTRUMENTATION_RUNNER +sourceSets { + main { + java.srcDir 'java/src/' + proto.srcDir 'proto/' } - compileOptions { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 - } - sourceSets { - main { - java.srcDir 'java/src/' - proto.srcDir 'proto/' - } - // TODO(b/161205849): Re-enable this test once icing nativeLib is no longer being built - // inside appsearch:appsearch. - //androidTest.java.srcDir 'java/tests/instrumentation/' - } - namespace "com.google.android.icing" -} - -// This project has no device tests, skip building it -androidComponents { - beforeVariants(selector().withName("debug"), { variantBuilder -> - variantBuilder.enableAndroidTest = false - }) } dependencies { - api('androidx.annotation:annotation:1.1.0') - - implementation('com.google.protobuf:protobuf-javalite:3.10.0') + compileOnly("androidx.annotation:annotation:1.1.0") + compileOnly(SdkHelperKt.getSdkDependency(project)) + implementation(libs.protobufLite) +} - androidTestImplementation(libs.testCore) - androidTestImplementation(libs.testRules) - androidTestImplementation(libs.truth) - androidTestImplementation(libs.kotlinBom) +afterEvaluate { + lint { + lintOptions { + // protobuf generates unannotated methods + disable("UnknownNullness") + } + } } protobuf { protoc { artifact = libs.protobufCompiler.get() } - generateProtoTasks { all().each { task -> - project.tasks.named("extractReleaseAnnotations").configure { - it.dependsOn(task) - } task.builtins { java { option 'lite' @@ -83,30 +59,6 @@ protobuf { } } -// Create export artifact for all variants (debug/release) for JarJaring -android.libraryVariants.all { variant -> - def variantName = variant.name - def suffix = variantName.capitalize() - def exportJarTask = tasks.register("exportJar${suffix}", Jar) { - archiveBaseName.set("icing-${variantName}") - - // The proto-lite dependency includes .proto files, which are not used by icing. When apps - // depend on appsearch as well as proto-lite directly, these files conflict since jarjar - // only renames the java classes. Remove them here since they are unused. - // Expand the jar and remove any .proto files. - from(zipTree(configurations.detachedConfiguration( - dependencies.create(libs.protobufLite.get())).getSingleFile())) { - exclude("**/*.proto") - } - - from files(variant.javaCompileProvider.get().destinationDir) - dependsOn variant.javaCompileProvider.get() - } - - def exportConfiguration = configurations.register("export${suffix}") - artifacts.add(exportConfiguration.name, exportJarTask.flatMap { it.archiveFile }) -} - androidx { mavenVersion = LibraryVersions.APPSEARCH } diff --git a/icing/file/persistent-hash-map.cc b/icing/file/persistent-hash-map.cc index 729b09a..558c242 100644 --- a/icing/file/persistent-hash-map.cc +++ b/icing/file/persistent-hash-map.cc @@ -27,6 +27,7 @@ #include "icing/absl_ports/str_cat.h" #include "icing/file/file-backed-vector.h" #include "icing/file/memory-mapped-file.h" +#include "icing/file/persistent-storage.h" #include "icing/util/crc32.h" #include "icing/util/status-macros.h" @@ -167,6 +168,8 @@ PersistentHashMap::~PersistentHashMap() { libtextclassifier3::Status PersistentHashMap::Put(std::string_view key, const void* value) { + SetDirty(); + ICING_RETURN_IF_ERROR(ValidateKey(key)); ICING_ASSIGN_OR_RETURN( int32_t bucket_idx, @@ -207,6 +210,7 @@ libtextclassifier3::Status PersistentHashMap::GetOrPut(std::string_view key, FindEntryIndexByKey(bucket_idx, key)); if (idx_pair.target_entry_index == Entry::kInvalidIndex) { // If not found, then insert new key value pair. + SetDirty(); return Insert(bucket_idx, key, next_value); } @@ -232,6 +236,8 @@ libtextclassifier3::Status PersistentHashMap::Get(std::string_view key, } libtextclassifier3::Status PersistentHashMap::Delete(std::string_view key) { + SetDirty(); + ICING_RETURN_IF_ERROR(ValidateKey(key)); ICING_ASSIGN_OR_RETURN( int32_t bucket_idx, @@ -514,6 +520,7 @@ PersistentHashMap::InitializeExistingFiles(const Filesystem& filesystem, << " to " << persistent_hash_map->options_.max_load_factor_percent; + persistent_hash_map->SetInfoDirty(); persistent_hash_map->info().max_load_factor_percent = persistent_hash_map->options_.max_load_factor_percent; ICING_RETURN_IF_ERROR( @@ -525,26 +532,50 @@ PersistentHashMap::InitializeExistingFiles(const Filesystem& filesystem, return persistent_hash_map; } -libtextclassifier3::Status PersistentHashMap::PersistStoragesToDisk() { +libtextclassifier3::Status PersistentHashMap::PersistStoragesToDisk( + bool force) { + if (!force && !is_storage_dirty()) { + return libtextclassifier3::Status::OK; + } + ICING_RETURN_IF_ERROR(bucket_storage_->PersistToDisk()); ICING_RETURN_IF_ERROR(entry_storage_->PersistToDisk()); ICING_RETURN_IF_ERROR(kv_storage_->PersistToDisk()); + is_storage_dirty_ = false; return libtextclassifier3::Status::OK; } -libtextclassifier3::Status PersistentHashMap::PersistMetadataToDisk() { +libtextclassifier3::Status PersistentHashMap::PersistMetadataToDisk( + bool force) { + // We can skip persisting metadata to disk only if both info and storage are + // clean. + if (!force && !is_info_dirty() && !is_storage_dirty()) { + return libtextclassifier3::Status::OK; + } + // Changes should have been applied to the underlying file when using // MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, but call msync() as an // extra safety step to ensure they are written out. - return metadata_mmapped_file_->PersistToDisk(); + ICING_RETURN_IF_ERROR(metadata_mmapped_file_->PersistToDisk()); + is_info_dirty_ = false; + return libtextclassifier3::Status::OK; } -libtextclassifier3::StatusOr<Crc32> PersistentHashMap::ComputeInfoChecksum() { +libtextclassifier3::StatusOr<Crc32> PersistentHashMap::ComputeInfoChecksum( + bool force) { + if (!force && !is_info_dirty()) { + return Crc32(crcs().component_crcs.info_crc); + } + return info().ComputeChecksum(); } -libtextclassifier3::StatusOr<Crc32> -PersistentHashMap::ComputeStoragesChecksum() { +libtextclassifier3::StatusOr<Crc32> PersistentHashMap::ComputeStoragesChecksum( + bool force) { + if (!force && !is_storage_dirty()) { + return Crc32(crcs().component_crcs.storages_crc); + } + // Compute crcs ICING_ASSIGN_OR_RETURN(Crc32 bucket_storage_crc, bucket_storage_->ComputeChecksum()); @@ -602,6 +633,8 @@ libtextclassifier3::Status PersistentHashMap::CopyEntryValue( libtextclassifier3::Status PersistentHashMap::Insert(int32_t bucket_idx, std::string_view key, const void* value) { + SetDirty(); + // If entry_storage_->num_elements() + 1 exceeds options_.max_num_entries, // then return error. // We compute max_file_size of 3 storages by options_.max_num_entries. Since @@ -655,6 +688,8 @@ libtextclassifier3::Status PersistentHashMap::RehashIfNecessary( return libtextclassifier3::Status::OK; } + SetDirty(); + // Resize and reset buckets. ICING_RETURN_IF_ERROR( bucket_storage_->Set(0, new_num_bucket, Bucket(Entry::kInvalidIndex))); diff --git a/icing/file/persistent-hash-map.h b/icing/file/persistent-hash-map.h index 845b22a..5f7999d 100644 --- a/icing/file/persistent-hash-map.h +++ b/icing/file/persistent-hash-map.h @@ -394,7 +394,9 @@ class PersistentHashMap : public PersistentStorage { std::move(metadata_mmapped_file))), bucket_storage_(std::move(bucket_storage)), entry_storage_(std::move(entry_storage)), - kv_storage_(std::move(kv_storage)) {} + kv_storage_(std::move(kv_storage)), + is_info_dirty_(false), + is_storage_dirty_(false) {} static libtextclassifier3::StatusOr<std::unique_ptr<PersistentHashMap>> InitializeNewFiles(const Filesystem& filesystem, std::string&& working_path, @@ -409,20 +411,20 @@ class PersistentHashMap : public PersistentStorage { // Returns: // - OK on success // - INTERNAL_ERROR on I/O error - libtextclassifier3::Status PersistStoragesToDisk() override; + libtextclassifier3::Status PersistStoragesToDisk(bool force) override; // Flushes contents of metadata file. // // Returns: // - OK on success // - INTERNAL_ERROR on I/O error - libtextclassifier3::Status PersistMetadataToDisk() override; + libtextclassifier3::Status PersistMetadataToDisk(bool force) override; // Computes and returns Info checksum. // // Returns: // - Crc of the Info on success - libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum() override; + libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum(bool force) override; // Computes and returns all storages checksum. Checksums of bucket_storage_, // entry_storage_ and kv_storage_ will be combined together by XOR. @@ -430,7 +432,8 @@ class PersistentHashMap : public PersistentStorage { // Returns: // - Crc of all storages on success // - INTERNAL_ERROR if any data inconsistency - libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum() override; + libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum( + bool force) override; // Find the index of the target entry (that contains the key) from a bucket // (specified by bucket index). Also return the previous entry index, since @@ -496,6 +499,17 @@ class PersistentHashMap : public PersistentStorage { kInfoMetadataFileOffset); } + void SetInfoDirty() { is_info_dirty_ = true; } + // When storage is dirty, we have to set info dirty as well. So just expose + // SetDirty to set both. + void SetDirty() { + is_info_dirty_ = true; + is_storage_dirty_ = true; + } + + bool is_info_dirty() const { return is_info_dirty_; } + bool is_storage_dirty() const { return is_storage_dirty_; } + Options options_; std::unique_ptr<MemoryMappedFile> metadata_mmapped_file_; @@ -504,6 +518,9 @@ class PersistentHashMap : public PersistentStorage { std::unique_ptr<FileBackedVector<Bucket>> bucket_storage_; std::unique_ptr<FileBackedVector<Entry>> entry_storage_; std::unique_ptr<FileBackedVector<char>> kv_storage_; + + bool is_info_dirty_; + bool is_storage_dirty_; }; } // namespace lib diff --git a/icing/file/persistent-storage.h b/icing/file/persistent-storage.h index 727cae9..9cb5e4d 100644 --- a/icing/file/persistent-storage.h +++ b/icing/file/persistent-storage.h @@ -148,8 +148,9 @@ class PersistentStorage { return libtextclassifier3::Status::OK; } - ICING_RETURN_IF_ERROR(UpdateChecksumsInternal()); - ICING_RETURN_IF_ERROR(PersistMetadataToDisk()); + ICING_RETURN_IF_ERROR(UpdateChecksumsInternal(/*force=*/true)); + ICING_RETURN_IF_ERROR(PersistStoragesToDisk(/*force=*/true)); + ICING_RETURN_IF_ERROR(PersistMetadataToDisk(/*force=*/true)); is_initialized_ = true; return libtextclassifier3::Status::OK; @@ -184,38 +185,52 @@ class PersistentStorage { // 2) Updates all checksums by new data. // 3) Flushes metadata. // + // Force flag will be passed down to PersistMetadataToDisk, + // PersistStoragesToDisk, ComputeInfoChecksum, ComputeStoragesChecksum. + // - If force == true, then performs actual persisting operations/recomputes + // the checksum. + // - Otherwise, the derived class can decide itself whether skipping + // persisting operations/doing lazy checksum recomputing if the storage is + // not dirty. + // // Returns: // - OK on success // - FAILED_PRECONDITION_ERROR if PersistentStorage is uninitialized // - Any errors from PersistStoragesToDisk, UpdateChecksums, // PersistMetadataToDisk, depending on actual implementation - libtextclassifier3::Status PersistToDisk() { + libtextclassifier3::Status PersistToDisk(bool force = false) { if (!is_initialized_) { return absl_ports::FailedPreconditionError(absl_ports::StrCat( "PersistentStorage ", working_path_, " not initialized")); } - ICING_RETURN_IF_ERROR(PersistStoragesToDisk()); - ICING_RETURN_IF_ERROR(UpdateChecksums()); - ICING_RETURN_IF_ERROR(PersistMetadataToDisk()); + ICING_RETURN_IF_ERROR(UpdateChecksumsInternal(force)); + ICING_RETURN_IF_ERROR(PersistStoragesToDisk(force)); + ICING_RETURN_IF_ERROR(PersistMetadataToDisk(force)); return libtextclassifier3::Status::OK; } // Updates checksums of all components and returns the overall crc (all_crc) // of the persistent storage. // + // Force flag will be passed down ComputeInfoChecksum, + // ComputeStoragesChecksum. + // - If force == true, then recomputes the checksum. + // - Otherwise, the derived class can decide itself whether doing lazy + // checksum recomputing if the storage is not dirty. + // // Returns: // - Overall crc of the persistent storage on success // - FAILED_PRECONDITION_ERROR if PersistentStorage is uninitialized // - Any errors from ComputeInfoChecksum, ComputeStoragesChecksum, depending // on actual implementation - libtextclassifier3::StatusOr<Crc32> UpdateChecksums() { + libtextclassifier3::StatusOr<Crc32> UpdateChecksums(bool force = false) { if (!is_initialized_) { return absl_ports::FailedPreconditionError(absl_ports::StrCat( "PersistentStorage ", working_path_, " not initialized")); } - return UpdateChecksumsInternal(); + return UpdateChecksumsInternal(force); } protected: @@ -234,33 +249,41 @@ class PersistentStorage { // Returns: // - OK on success // - Any other errors, depending on actual implementation - virtual libtextclassifier3::Status PersistMetadataToDisk() = 0; + virtual libtextclassifier3::Status PersistMetadataToDisk(bool force) = 0; // Flushes contents of all storages to underlying files. // // Returns: // - OK on success // - Any other errors, depending on actual implementation - virtual libtextclassifier3::Status PersistStoragesToDisk() = 0; + virtual libtextclassifier3::Status PersistStoragesToDisk(bool force) = 0; // Computes and returns Info checksum. + // - If force = true, then recompute the entire checksum. + // - Otherwise, the derived class can decide itself whether doing lazy + // checksum computing if the storage is not dirty. // // This function will be mainly called by UpdateChecksums. // // Returns: // - Crc of the Info on success // - Any other errors, depending on actual implementation - virtual libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum() = 0; + virtual libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum( + bool force) = 0; // Computes and returns all storages checksum. If there are multiple storages, // usually we XOR their checksums together to a single checksum. + // - If force = true, then recompute the entire checksum. + // - Otherwise, the derived class can decide itself whether doing lazy + // checksum computing if the storage is not dirty. // // This function will be mainly called by UpdateChecksums. // // Returns: // - Crc of all storages on success // - Any other errors from depending on actual implementation - virtual libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum() = 0; + virtual libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum( + bool force) = 0; // Returns the Crcs instance reference. The derived class can either own a // concrete Crcs instance, or reinterpret_cast the memory-mapped region to @@ -292,11 +315,18 @@ class PersistentStorage { // - Overall crc of the persistent storage on success // - Any errors from ComputeInfoChecksum, ComputeStoragesChecksum, depending // on actual implementation - libtextclassifier3::StatusOr<Crc32> UpdateChecksumsInternal() { + libtextclassifier3::StatusOr<Crc32> UpdateChecksumsInternal(bool force) { Crcs& crcs_ref = crcs(); // Compute and update storages + info checksums. - ICING_ASSIGN_OR_RETURN(Crc32 info_crc, ComputeInfoChecksum()); - ICING_ASSIGN_OR_RETURN(Crc32 storages_crc, ComputeStoragesChecksum()); + ICING_ASSIGN_OR_RETURN(Crc32 info_crc, ComputeInfoChecksum(force)); + ICING_ASSIGN_OR_RETURN(Crc32 storages_crc, ComputeStoragesChecksum(force)); + if (crcs_ref.component_crcs.info_crc == info_crc.Get() && + crcs_ref.component_crcs.storages_crc == storages_crc.Get()) { + // If info and storages crc haven't changed, then we don't have to update + // checksums. + return Crc32(crcs_ref.all_crc); + } + crcs_ref.component_crcs.info_crc = info_crc.Get(); crcs_ref.component_crcs.storages_crc = storages_crc.Get(); @@ -318,12 +348,13 @@ class PersistentStorage { return absl_ports::FailedPreconditionError("Invalid all crc"); } - ICING_ASSIGN_OR_RETURN(Crc32 info_crc, ComputeInfoChecksum()); + ICING_ASSIGN_OR_RETURN(Crc32 info_crc, ComputeInfoChecksum(/*force=*/true)); if (crcs_ref.component_crcs.info_crc != info_crc.Get()) { return absl_ports::FailedPreconditionError("Invalid info crc"); } - ICING_ASSIGN_OR_RETURN(Crc32 storages_crc, ComputeStoragesChecksum()); + ICING_ASSIGN_OR_RETURN(Crc32 storages_crc, + ComputeStoragesChecksum(/*force=*/true)); if (crcs_ref.component_crcs.storages_crc != storages_crc.Get()) { return absl_ports::FailedPreconditionError("Invalid storages crc"); } diff --git a/icing/file/posting_list/flash-index-storage.cc b/icing/file/posting_list/flash-index-storage.cc index cd7ac12..21fea8a 100644 --- a/icing/file/posting_list/flash-index-storage.cc +++ b/icing/file/posting_list/flash-index-storage.cc @@ -487,6 +487,9 @@ libtextclassifier3::Status FlashIndexStorage::FreePostingList( PostingListHolder&& holder) { ICING_ASSIGN_OR_RETURN(IndexBlock block, GetIndexBlock(holder.id.block_index())); + if (block.posting_list_bytes() == max_posting_list_bytes()) { + block.SetNextBlockIndex(kInvalidBlockIndex); + } uint32_t posting_list_bytes = block.posting_list_bytes(); int best_block_info_index = FindBestIndexBlockInfo(posting_list_bytes); diff --git a/icing/file/version-util.cc b/icing/file/version-util.cc index f477072..7684262 100644 --- a/icing/file/version-util.cc +++ b/icing/file/version-util.cc @@ -102,6 +102,44 @@ StateChange GetVersionStateChange(const VersionInfo& existing_version_info, } } +bool ShouldRebuildDerivedFiles(const VersionInfo& existing_version_info, + int32_t curr_version) { + StateChange state_change = + GetVersionStateChange(existing_version_info, curr_version); + switch (state_change) { + case StateChange::kCompatible: + return false; + case StateChange::kUndetermined: + [[fallthrough]]; + case StateChange::kRollBack: + [[fallthrough]]; + case StateChange::kRollForward: + [[fallthrough]]; + case StateChange::kVersionZeroRollForward: + [[fallthrough]]; + case StateChange::kVersionZeroUpgrade: + return true; + case StateChange::kUpgrade: + break; + } + + bool should_rebuild = false; + int32_t existing_version = existing_version_info.version; + while (existing_version < curr_version) { + switch (existing_version) { + case 1: { + // version 1 -> version 2 upgrade, no need to rebuild + break; + } + default: + // This should not happen. Rebuild anyway if unsure. + should_rebuild |= true; + } + ++existing_version; + } + return should_rebuild; +} + } // namespace version_util } // namespace lib diff --git a/icing/file/version-util.h b/icing/file/version-util.h index 7fa7fbd..30c457d 100644 --- a/icing/file/version-util.h +++ b/icing/file/version-util.h @@ -28,9 +28,16 @@ namespace lib { namespace version_util { // - Version 0: Android T. Can be identified only by flash index magic. -// - Version 1: mainline release 2023-06. -inline static constexpr int32_t kVersion = 1; +// - Version 1: Android U release 2023-06. +// - Version 2: Android U 1st mainline release 2023-09. Schema is compatible +// with version 1. +// TODO(b/288969109): bump kVersion to 2 before finalizing the 1st Android U +// mainline release. +// LINT.IfChange(kVersion) +inline static constexpr int32_t kVersion = 2; +// LINT.ThenChange(//depot/google3/icing/schema/schema-store.cc:min_overlay_version_compatibility) inline static constexpr int32_t kVersionOne = 1; +inline static constexpr int32_t kVersionTwo = 2; inline static constexpr int kVersionZeroFlashIndexMagic = 0x6dfba6ae; @@ -89,6 +96,16 @@ libtextclassifier3::Status WriteVersion(const Filesystem& filesystem, StateChange GetVersionStateChange(const VersionInfo& existing_version_info, int32_t curr_version = kVersion); +// Helper method to determine whether Icing should rebuild all derived files. +// Sometimes it is not required to rebuild derived files when +// roll-forward/upgrading. This function "encodes" upgrade paths and checks if +// the roll-forward/upgrading requires derived files to be rebuilt or not. +// +// REQUIRES: curr_version > 0. We implement version checking in version 1, so +// the callers (except unit tests) will always use a version # greater than 0. +bool ShouldRebuildDerivedFiles(const VersionInfo& existing_version_info, + int32_t curr_version = kVersion); + } // namespace version_util } // namespace lib diff --git a/icing/file/version-util_test.cc b/icing/file/version-util_test.cc index 78cdb7d..e94c351 100644 --- a/icing/file/version-util_test.cc +++ b/icing/file/version-util_test.cc @@ -32,6 +32,8 @@ namespace version_util { namespace { using ::testing::Eq; +using ::testing::IsFalse; +using ::testing::IsTrue; struct VersionUtilReadVersionTestParam { std::optional<VersionInfo> existing_version_info; @@ -339,6 +341,14 @@ INSTANTIATE_TEST_SUITE_P( /*curr_version_in=*/2, /*expected_state_change_in=*/StateChange::kRollForward), + // - version 1, max_version 2 + // - Current version = 3 + // - Result: roll forward + VersionUtilStateChangeTestParam( + /*existing_version_info_in=*/VersionInfo(1, 2), + /*curr_version_in=*/3, + /*expected_state_change_in=*/StateChange::kRollForward), + // - version 1, max_version 3 // - Current version = 2 // - Result: roll forward @@ -379,6 +389,84 @@ INSTANTIATE_TEST_SUITE_P( /*curr_version_in=*/2, /*expected_state_change_in=*/StateChange::kRollBack))); +TEST(VersionUtilTest, ShouldRebuildDerivedFilesUndeterminedVersion) { + EXPECT_THAT( + ShouldRebuildDerivedFiles(VersionInfo(-1, -1), /*curr_version=*/1), + IsTrue()); + EXPECT_THAT( + ShouldRebuildDerivedFiles(VersionInfo(-1, -1), /*curr_version=*/2), + IsTrue()); +} + +TEST(VersionUtilTest, ShouldRebuildDerivedFilesVersionZeroUpgrade) { + // 0 -> 1 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(0, 0), /*curr_version=*/1), + IsTrue()); + + // 0 -> 2 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(0, 0), /*curr_version=*/2), + IsTrue()); +} + +TEST(VersionUtilTest, ShouldRebuildDerivedFilesVersionZeroRollForward) { + // (1 -> 0), 0 -> 1 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(0, 1), /*curr_version=*/1), + IsTrue()); + + // (1 -> 0), 0 -> 2 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(0, 1), /*curr_version=*/2), + IsTrue()); + + // (2 -> 0), 0 -> 1 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(0, 2), /*curr_version=*/1), + IsTrue()); +} + +TEST(VersionUtilTest, ShouldRebuildDerivedFilesRollBack) { + // 2 -> 1 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(2, 2), /*curr_version=*/1), + IsTrue()); + + // 3 -> 1 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(3, 3), /*curr_version=*/1), + IsTrue()); + + // (3 -> 2), 2 -> 1 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(2, 3), /*curr_version=*/1), + IsTrue()); +} + +TEST(VersionUtilTest, ShouldRebuildDerivedFilesRollForward) { + // (2 -> 1), 1 -> 2 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(1, 2), /*curr_version=*/2), + IsTrue()); + + // (2 -> 1), 1 -> 3 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(1, 2), /*curr_version=*/3), + IsTrue()); + + // (3 -> 1), 1 -> 2 + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(1, 3), /*curr_version=*/2), + IsTrue()); +} + +TEST(VersionUtilTest, ShouldRebuildDerivedFilesCompatible) { + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(2, 2), /*curr_version=*/2), + IsFalse()); + + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(2, 3), /*curr_version=*/2), + IsFalse()); +} + +TEST(VersionUtilTest, ShouldRebuildDerivedFilesUpgrade) { + // Unlike other state changes, upgrade depends on the actual "encoded path". + + // kVersionOne -> kVersionTwo + EXPECT_THAT(ShouldRebuildDerivedFiles(VersionInfo(kVersionOne, kVersionOne), + /*curr_version=*/kVersionTwo), + IsFalse()); +} + } // namespace } // namespace version_util diff --git a/icing/icing-search-engine.cc b/icing/icing-search-engine.cc index 2cdf930..6680dae 100644 --- a/icing/icing-search-engine.cc +++ b/icing/icing-search-engine.cc @@ -42,8 +42,8 @@ #include "icing/index/numeric/integer-index.h" #include "icing/index/string-section-indexing-handler.h" #include "icing/join/join-processor.h" +#include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id-join-indexing-handler.h" -#include "icing/join/qualified-id-type-joinable-index.h" #include "icing/legacy/index/icing-filesystem.h" #include "icing/portable/endian.h" #include "icing/proto/debug.pb.h" @@ -141,6 +141,15 @@ libtextclassifier3::Status ValidateResultSpec( "ResultSpecProto.num_total_bytes_per_page_threshold cannot be " "non-positive."); } + if (result_spec.max_joined_children_per_parent_to_return() < 0) { + return absl_ports::InvalidArgumentError( + "ResultSpecProto.max_joined_children_per_parent_to_return cannot be " + "negative."); + } + if (result_spec.num_to_score() <= 0) { + return absl_ports::InvalidArgumentError( + "ResultSpecProto.num_to_score cannot be non-positive."); + } // Validate ResultGroupings. std::unordered_set<int32_t> unique_entry_ids; ResultSpecProto::ResultGroupingType result_grouping_type = @@ -583,8 +592,10 @@ libtextclassifier3::Status IcingSearchEngine::InitializeMembers( filesystem_.get(), MakeSchemaDirectoryPath(options_.base_dir()), version_state_change, version_util::kVersion)); - // Step 2: discard all derived data - ICING_RETURN_IF_ERROR(DiscardDerivedFiles()); + // Step 2: discard all derived data if needed rebuild. + if (version_util::ShouldRebuildDerivedFiles(version_info)) { + ICING_RETURN_IF_ERROR(DiscardDerivedFiles()); + } // Step 3: update version file version_util::VersionInfo new_version_info( @@ -621,8 +632,8 @@ libtextclassifier3::Status IcingSearchEngine::InitializeMembers( if (!filesystem_->DeleteDirectoryRecursively(doc_store_dir.c_str()) || !filesystem_->DeleteDirectoryRecursively(index_dir.c_str()) || !IntegerIndex::Discard(*filesystem_, integer_index_dir).ok() || - !QualifiedIdTypeJoinableIndex::Discard(*filesystem_, - qualified_id_join_index_dir) + !QualifiedIdJoinIndex::Discard(*filesystem_, + qualified_id_join_index_dir) .ok()) { return absl_ports::InternalError(absl_ports::StrCat( "Could not delete directories: ", index_dir, ", ", integer_index_dir, @@ -643,7 +654,9 @@ libtextclassifier3::Status IcingSearchEngine::InitializeMembers( // We're going to need to build the index from scratch. So just delete its // directory now. // Discard index directory and instantiate a new one. - Index::Options index_options(index_dir, options_.index_merge_size()); + Index::Options index_options(index_dir, options_.index_merge_size(), + options_.lite_index_sort_at_indexing(), + options_.lite_index_sort_size()); if (!filesystem_->DeleteDirectoryRecursively(index_dir.c_str()) || !filesystem_->CreateDirectoryRecursively(index_dir.c_str())) { return absl_ports::InternalError( @@ -661,16 +674,17 @@ libtextclassifier3::Status IcingSearchEngine::InitializeMembers( ICING_ASSIGN_OR_RETURN( integer_index_, IntegerIndex::Create(*filesystem_, std::move(integer_index_dir), + options_.integer_index_bucket_split_threshold(), options_.pre_mapping_fbv())); // Discard qualified id join index directory and instantiate a new one. std::string qualified_id_join_index_dir = MakeQualifiedIdJoinIndexWorkingPath(options_.base_dir()); - ICING_RETURN_IF_ERROR(QualifiedIdTypeJoinableIndex::Discard( + ICING_RETURN_IF_ERROR(QualifiedIdJoinIndex::Discard( *filesystem_, qualified_id_join_index_dir)); ICING_ASSIGN_OR_RETURN( qualified_id_join_index_, - QualifiedIdTypeJoinableIndex::Create( + QualifiedIdJoinIndex::Create( *filesystem_, std::move(qualified_id_join_index_dir), options_.pre_mapping_fbv(), options_.use_persistent_hash_map())); @@ -765,11 +779,12 @@ libtextclassifier3::Status IcingSearchEngine::InitializeDocumentStore( } ICING_ASSIGN_OR_RETURN( DocumentStore::CreateResult create_result, - DocumentStore::Create(filesystem_.get(), document_dir, clock_.get(), - schema_store_.get(), - force_recovery_and_revalidate_documents, - options_.document_store_namespace_id_fingerprint(), - options_.compression_level(), initialize_stats)); + DocumentStore::Create( + filesystem_.get(), document_dir, clock_.get(), schema_store_.get(), + force_recovery_and_revalidate_documents, + options_.document_store_namespace_id_fingerprint(), + options_.pre_mapping_fbv(), options_.use_persistent_hash_map(), + options_.compression_level(), initialize_stats)); document_store_ = std::move(create_result.document_store); return libtextclassifier3::Status::OK; @@ -785,7 +800,9 @@ libtextclassifier3::Status IcingSearchEngine::InitializeIndex( return absl_ports::InternalError( absl_ports::StrCat("Could not create directory: ", index_dir)); } - Index::Options index_options(index_dir, options_.index_merge_size()); + Index::Options index_options(index_dir, options_.index_merge_size(), + options_.lite_index_sort_at_indexing(), + options_.lite_index_sort_size()); // Term index InitializeStatsProto::RecoveryCause index_recovery_cause; @@ -816,8 +833,10 @@ libtextclassifier3::Status IcingSearchEngine::InitializeIndex( std::string integer_index_dir = MakeIntegerIndexWorkingPath(options_.base_dir()); InitializeStatsProto::RecoveryCause integer_index_recovery_cause; - auto integer_index_or = IntegerIndex::Create(*filesystem_, integer_index_dir, - options_.pre_mapping_fbv()); + auto integer_index_or = + IntegerIndex::Create(*filesystem_, integer_index_dir, + options_.integer_index_bucket_split_threshold(), + options_.pre_mapping_fbv()); if (!integer_index_or.ok()) { ICING_RETURN_IF_ERROR( IntegerIndex::Discard(*filesystem_, integer_index_dir)); @@ -828,6 +847,7 @@ libtextclassifier3::Status IcingSearchEngine::InitializeIndex( ICING_ASSIGN_OR_RETURN( integer_index_, IntegerIndex::Create(*filesystem_, std::move(integer_index_dir), + options_.integer_index_bucket_split_threshold(), options_.pre_mapping_fbv())); } else { // Integer index was created fine. @@ -842,11 +862,11 @@ libtextclassifier3::Status IcingSearchEngine::InitializeIndex( std::string qualified_id_join_index_dir = MakeQualifiedIdJoinIndexWorkingPath(options_.base_dir()); InitializeStatsProto::RecoveryCause qualified_id_join_index_recovery_cause; - auto qualified_id_join_index_or = QualifiedIdTypeJoinableIndex::Create( + auto qualified_id_join_index_or = QualifiedIdJoinIndex::Create( *filesystem_, qualified_id_join_index_dir, options_.pre_mapping_fbv(), options_.use_persistent_hash_map()); if (!qualified_id_join_index_or.ok()) { - ICING_RETURN_IF_ERROR(QualifiedIdTypeJoinableIndex::Discard( + ICING_RETURN_IF_ERROR(QualifiedIdJoinIndex::Discard( *filesystem_, qualified_id_join_index_dir)); qualified_id_join_index_recovery_cause = InitializeStatsProto::IO_ERROR; @@ -854,7 +874,7 @@ libtextclassifier3::Status IcingSearchEngine::InitializeIndex( // Try recreating it from scratch and rebuild everything. ICING_ASSIGN_OR_RETURN( qualified_id_join_index_, - QualifiedIdTypeJoinableIndex::Create( + QualifiedIdJoinIndex::Create( *filesystem_, std::move(qualified_id_join_index_dir), options_.pre_mapping_fbv(), options_.use_persistent_hash_map())); } else { @@ -2124,7 +2144,7 @@ IcingSearchEngine::QueryScoringResults IcingSearchEngine::ProcessQueryAndScore( std::move(scoring_processor_or).ValueOrDie(); std::vector<ScoredDocumentHit> scored_document_hits = scoring_processor->Score(std::move(query_results.root_iterator), - performance_configuration_.num_to_score, + result_spec.num_to_score(), &query_results.query_term_iterators); int64_t scoring_latency_ms = component_timer->GetElapsedMilliseconds(); @@ -2241,8 +2261,7 @@ IcingSearchEngine::OptimizeDocumentStore(OptimizeStatsProto* optimize_stats) { // Copies valid document data to tmp directory libtextclassifier3::StatusOr<std::vector<DocumentId>> document_id_old_to_new_or = document_store_->OptimizeInto( - temporary_document_dir, language_segmenter_.get(), - options_.document_store_namespace_id_fingerprint(), optimize_stats); + temporary_document_dir, language_segmenter_.get(), optimize_stats); // Handles error if any if (!document_id_old_to_new_or.ok()) { @@ -2280,6 +2299,7 @@ IcingSearchEngine::OptimizeDocumentStore(OptimizeStatsProto* optimize_stats) { filesystem_.get(), current_document_dir, clock_.get(), schema_store_.get(), /*force_recovery_and_revalidate_documents=*/false, options_.document_store_namespace_id_fingerprint(), + options_.pre_mapping_fbv(), options_.use_persistent_hash_map(), options_.compression_level(), /*initialize_stats=*/nullptr); // TODO(b/144458732): Implement a more robust version of // TC_ASSIGN_OR_RETURN that can support error logging. @@ -2307,6 +2327,7 @@ IcingSearchEngine::OptimizeDocumentStore(OptimizeStatsProto* optimize_stats) { filesystem_.get(), current_document_dir, clock_.get(), schema_store_.get(), /*force_recovery_and_revalidate_documents=*/false, options_.document_store_namespace_id_fingerprint(), + options_.pre_mapping_fbv(), options_.use_persistent_hash_map(), options_.compression_level(), /*initialize_stats=*/nullptr); if (!create_result_or.ok()) { // Unable to create DocumentStore from the new file. Mark as uninitialized @@ -2471,13 +2492,12 @@ IcingSearchEngine::CreateDataIndexingHandlers() { clock_.get(), integer_index_.get())); handlers.push_back(std::move(integer_section_indexing_handler)); - // Qualified id joinable property index handler + // Qualified id join index handler ICING_ASSIGN_OR_RETURN(std::unique_ptr<QualifiedIdJoinIndexingHandler> - qualified_id_joinable_property_indexing_handler, + qualified_id_join_indexing_handler, QualifiedIdJoinIndexingHandler::Create( clock_.get(), qualified_id_join_index_.get())); - handlers.push_back( - std::move(qualified_id_joinable_property_indexing_handler)); + handlers.push_back(std::move(qualified_id_join_indexing_handler)); return handlers; } diff --git a/icing/icing-search-engine.h b/icing/icing-search-engine.h index 15da142..d9d5ff6 100644 --- a/icing/icing-search-engine.h +++ b/icing/icing-search-engine.h @@ -31,7 +31,7 @@ #include "icing/index/numeric/numeric-index.h" #include "icing/jni/jni-cache.h" #include "icing/join/join-children-fetcher.h" -#include "icing/join/qualified-id-type-joinable-index.h" +#include "icing/join/qualified-id-join-index.h" #include "icing/legacy/index/icing-filesystem.h" #include "icing/performance-configuration.h" #include "icing/proto/debug.pb.h" @@ -479,7 +479,7 @@ class IcingSearchEngine { ICING_GUARDED_BY(mutex_); // Storage for all join qualified ids from the document store. - std::unique_ptr<QualifiedIdTypeJoinableIndex> qualified_id_join_index_ + std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index_ ICING_GUARDED_BY(mutex_); // Pointer to JNI class references diff --git a/icing/icing-search-engine_benchmark.cc b/icing/icing-search-engine_benchmark.cc index fb44595..354d11c 100644 --- a/icing/icing-search-engine_benchmark.cc +++ b/icing/icing-search-engine_benchmark.cc @@ -37,6 +37,7 @@ #include "icing/join/join-processor.h" #include "icing/proto/document.pb.h" #include "icing/proto/initialize.pb.h" +#include "icing/proto/persist.pb.h" #include "icing/proto/reset.pb.h" #include "icing/proto/schema.pb.h" #include "icing/proto/scoring.pb.h" @@ -1214,6 +1215,58 @@ void BM_JoinQueryQualifiedId(benchmark::State& state) { } BENCHMARK(BM_JoinQueryQualifiedId); +void BM_PersistToDisk(benchmark::State& state) { + // Initialize the filesystem + std::string test_dir = GetTestTempDir() + "/icing/benchmark"; + Filesystem filesystem; + DestructibleDirectory ddir(filesystem, test_dir); + + // Create the schema. + std::default_random_engine random; + int num_types = kAvgNumNamespaces * kAvgNumTypes; + ExactStringPropertyGenerator property_generator; + SchemaGenerator<ExactStringPropertyGenerator> schema_generator( + /*num_properties=*/state.range(1), &property_generator); + SchemaProto schema = schema_generator.GenerateSchema(num_types); + EvenDistributionTypeSelector type_selector(schema); + + // Generate documents. + int num_docs = state.range(0); + std::vector<std::string> language = CreateLanguages(kLanguageSize, &random); + const std::vector<DocumentProto> random_docs = + GenerateRandomDocuments(&type_selector, num_docs, language); + + for (auto _ : state) { + state.PauseTiming(); + // Create the index. + IcingSearchEngineOptions options; + options.set_base_dir(test_dir); + options.set_index_merge_size(kIcingFullIndexSize); + options.set_use_persistent_hash_map(true); + std::unique_ptr<IcingSearchEngine> icing = + std::make_unique<IcingSearchEngine>(options); + + ASSERT_THAT(icing->Reset().status(), ProtoIsOk()); + ASSERT_THAT(icing->SetSchema(schema).status(), ProtoIsOk()); + + for (const DocumentProto& doc : random_docs) { + ASSERT_THAT(icing->Put(doc).status(), ProtoIsOk()); + } + + state.ResumeTiming(); + + ASSERT_THAT(icing->PersistToDisk(PersistType::FULL).status(), ProtoIsOk()); + + state.PauseTiming(); + icing.reset(); + ASSERT_TRUE(filesystem.DeleteDirectoryRecursively(test_dir.c_str())); + state.ResumeTiming(); + } +} +BENCHMARK(BM_PersistToDisk) + // Arguments: num_indexed_documents, num_sections + ->ArgPair(1024, 5); + } // namespace } // namespace lib diff --git a/icing/icing-search-engine_initialization_test.cc b/icing/icing-search-engine_initialization_test.cc index 74cc78f..b4853b4 100644 --- a/icing/icing-search-engine_initialization_test.cc +++ b/icing/icing-search-engine_initialization_test.cc @@ -34,8 +34,8 @@ #include "icing/jni/jni-cache.h" #include "icing/join/doc-join-info.h" #include "icing/join/join-processor.h" +#include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id-join-indexing-handler.h" -#include "icing/join/qualified-id-type-joinable-index.h" #include "icing/legacy/index/icing-filesystem.h" #include "icing/legacy/index/icing-mock-filesystem.h" #include "icing/portable/endian.h" @@ -88,6 +88,7 @@ using ::testing::Eq; using ::testing::HasSubstr; using ::testing::IsEmpty; using ::testing::Matcher; +using ::testing::Ne; using ::testing::Return; using ::testing::SizeIs; @@ -1060,13 +1061,14 @@ TEST_F(IcingSearchEngineInitializationTest, // Puts message2 into DocumentStore but doesn't index it. ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(filesystem(), GetDocumentDir(), &fake_clock, - schema_store.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + filesystem(), GetDocumentDir(), &fake_clock, schema_store.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); @@ -1493,6 +1495,108 @@ TEST_F(IcingSearchEngineInitializationTest, RecoverFromCorruptIntegerIndex) { } TEST_F(IcingSearchEngineInitializationTest, + RecoverFromIntegerIndexBucketSplitThresholdChange) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty( + PropertyConfigBuilder() + .SetName("indexableInteger") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + + DocumentProto message = + DocumentBuilder() + .SetKey("namespace", "message/1") + .SetSchema("Message") + .AddInt64Property("indexableInteger", 123) + .SetCreationTimestampMs(kDefaultCreationTimestampMs) + .Build(); + + // 1. Create an index with a message document. + { + TestIcingSearchEngine icing( + GetDefaultIcingOptions(), std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), std::make_unique<FakeClock>(), + GetTestJniCache()); + + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status(), ProtoIsOk()); + + EXPECT_THAT(icing.Put(message).status(), ProtoIsOk()); + } + + // 2. Create the index again with different + // integer_index_bucket_split_threshold. This should trigger index + // restoration. + { + // Mock filesystem to observe and check the behavior of all indices. + auto mock_filesystem = std::make_unique<MockFilesystem>(); + EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively(_)) + .WillRepeatedly(DoDefault()); + // Ensure term index directory should never be discarded. + EXPECT_CALL(*mock_filesystem, + DeleteDirectoryRecursively(EndsWith("/index_dir"))) + .Times(0); + // Ensure integer index directory should be discarded once, and Clear() + // should never be called (i.e. storage sub directory + // "*/integer_index_dir/*" should never be discarded) since we start it from + // scratch. + EXPECT_CALL(*mock_filesystem, + DeleteDirectoryRecursively(EndsWith("/integer_index_dir"))) + .Times(1); + EXPECT_CALL(*mock_filesystem, + DeleteDirectoryRecursively(HasSubstr("/integer_index_dir/"))) + .Times(0); + // Ensure qualified id join index directory should never be discarded, and + // Clear() should never be called (i.e. storage sub directory + // "*/qualified_id_join_index_dir/*" should never be discarded). + EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively( + EndsWith("/qualified_id_join_index_dir"))) + .Times(0); + EXPECT_CALL( + *mock_filesystem, + DeleteDirectoryRecursively(HasSubstr("/qualified_id_join_index_dir/"))) + .Times(0); + + static constexpr int32_t kNewIntegerIndexBucketSplitThreshold = 1000; + IcingSearchEngineOptions options = GetDefaultIcingOptions(); + ASSERT_THAT(kNewIntegerIndexBucketSplitThreshold, + Ne(options.integer_index_bucket_split_threshold())); + options.set_integer_index_bucket_split_threshold( + kNewIntegerIndexBucketSplitThreshold); + + TestIcingSearchEngine icing(options, std::move(mock_filesystem), + std::make_unique<IcingFilesystem>(), + std::make_unique<FakeClock>(), + GetTestJniCache()); + InitializeResultProto initialize_result = icing.Initialize(); + ASSERT_THAT(initialize_result.status(), ProtoIsOk()); + EXPECT_THAT(initialize_result.initialize_stats().index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + EXPECT_THAT( + initialize_result.initialize_stats().integer_index_restoration_cause(), + Eq(InitializeStatsProto::IO_ERROR)); + EXPECT_THAT(initialize_result.initialize_stats() + .qualified_id_join_index_restoration_cause(), + Eq(InitializeStatsProto::NONE)); + + // Verify integer index works normally + SearchSpecProto search_spec; + search_spec.set_query("indexableInteger == 123"); + search_spec.set_search_type( + SearchSpecProto::SearchType::EXPERIMENTAL_ICING_ADVANCED_QUERY); + search_spec.add_enabled_features(std::string(kNumericSearchFeature)); + + SearchResultProto results = + icing.Search(search_spec, ScoringSpecProto::default_instance(), + ResultSpecProto::default_instance()); + ASSERT_THAT(results.results(), SizeIs(1)); + EXPECT_THAT(results.results(0).document().uri(), Eq("message/1")); + } +} + +TEST_F(IcingSearchEngineInitializationTest, RecoverFromCorruptQualifiedIdJoinIndex) { // Test the following scenario: qualified id join index is corrupted (e.g. // checksum doesn't match). IcingSearchEngine should be able to recover @@ -1749,7 +1853,9 @@ TEST_F(IcingSearchEngineInitializationTest, RestoreIndexLoseTermIndex) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Index> index, Index::Create(Index::Options(GetIndexDir(), - /*index_merge_size=*/100), + /*index_merge_size=*/100, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/50), filesystem(), icing_filesystem())); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -2359,7 +2465,9 @@ TEST_F(IcingSearchEngineInitializationTest, std::unique_ptr<Index> index, Index::Create( Index::Options(GetIndexDir(), - /*index_merge_size=*/message.ByteSizeLong()), + /*index_merge_size=*/message.ByteSizeLong(), + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/8), filesystem(), icing_filesystem())); DocumentId original_last_added_doc_id = index->last_added_document_id(); index->set_last_added_document_id(original_last_added_doc_id + 1); @@ -2491,7 +2599,9 @@ TEST_F(IcingSearchEngineInitializationTest, std::unique_ptr<Index> index, Index::Create( Index::Options(GetIndexDir(), - /*index_merge_size=*/message.ByteSizeLong()), + /*index_merge_size=*/message.ByteSizeLong(), + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/8), filesystem(), icing_filesystem())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<DocHitInfoIterator> doc_hit_info_iter, @@ -2603,7 +2713,9 @@ TEST_F(IcingSearchEngineInitializationTest, std::unique_ptr<Index> index, Index::Create( Index::Options(GetIndexDir(), - /*index_merge_size=*/message.ByteSizeLong()), + /*index_merge_size=*/message.ByteSizeLong(), + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/8), filesystem(), icing_filesystem())); DocumentId original_last_added_doc_id = index->last_added_document_id(); index->set_last_added_document_id(original_last_added_doc_id + 1); @@ -2740,7 +2852,9 @@ TEST_F(IcingSearchEngineInitializationTest, std::unique_ptr<Index> index, Index::Create( Index::Options(GetIndexDir(), - /*index_merge_size=*/message.ByteSizeLong()), + /*index_merge_size=*/message.ByteSizeLong(), + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/8), filesystem(), icing_filesystem())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<DocHitInfoIterator> doc_hit_info_iter, @@ -2800,7 +2914,9 @@ TEST_F(IcingSearchEngineInitializationTest, Index::Create( // index merge size is not important here because we will manually // invoke merge below. - Index::Options(GetIndexDir(), /*index_merge_size=*/100), + Index::Options(GetIndexDir(), /*index_merge_size=*/100, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/50), filesystem(), icing_filesystem())); // Add hits for document 0 and merge. ASSERT_THAT(index->last_added_document_id(), kInvalidDocumentId); @@ -2876,7 +2992,9 @@ TEST_F(IcingSearchEngineInitializationTest, { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Index> index, - Index::Create(Index::Options(GetIndexDir(), /*index_merge_size=*/100), + Index::Create(Index::Options(GetIndexDir(), /*index_merge_size=*/100, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/50), filesystem(), icing_filesystem())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<DocHitInfoIterator> doc_hit_info_iter, @@ -2992,7 +3110,9 @@ TEST_F(IcingSearchEngineInitializationTest, std::unique_ptr<Index> index, Index::Create( Index::Options(GetIndexDir(), - /*index_merge_size=*/message.ByteSizeLong()), + /*index_merge_size=*/message.ByteSizeLong(), + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/8), filesystem(), icing_filesystem())); // Add hits for document 4 and merge. DocumentId original_last_added_doc_id = index->last_added_document_id(); @@ -3135,7 +3255,9 @@ TEST_F(IcingSearchEngineInitializationTest, { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Index> index, - Index::Create(Index::Options(GetIndexDir(), /*index_merge_size=*/100), + Index::Create(Index::Options(GetIndexDir(), /*index_merge_size=*/100, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/50), filesystem(), icing_filesystem())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<DocHitInfoIterator> doc_hit_info_iter, @@ -3197,6 +3319,7 @@ TEST_F(IcingSearchEngineInitializationTest, ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem, GetIntegerIndexDir(), + /*num_data_threshold_for_bucket_split=*/65536, /*pre_mapping_fbv=*/false)); // Add hits for document 0. ASSERT_THAT(integer_index->last_added_document_id(), kInvalidDocumentId); @@ -3376,6 +3499,7 @@ TEST_F(IcingSearchEngineInitializationTest, ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem, GetIntegerIndexDir(), + /*num_data_threshold_for_bucket_split=*/65536, /*pre_mapping_fbv=*/false)); // Add hits for document 4. DocumentId original_last_added_doc_id = @@ -3571,10 +3695,10 @@ TEST_F(IcingSearchEngineInitializationTest, { Filesystem filesystem; ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> qualified_id_join_index, - QualifiedIdTypeJoinableIndex::Create( - filesystem, GetQualifiedIdJoinIndexDir(), /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/false)); + std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index, + QualifiedIdJoinIndex::Create(filesystem, GetQualifiedIdJoinIndexDir(), + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false)); // Add data for document 0. ASSERT_THAT(qualified_id_join_index->last_added_document_id(), kInvalidDocumentId); @@ -3641,10 +3765,10 @@ TEST_F(IcingSearchEngineInitializationTest, { Filesystem filesystem; ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> qualified_id_join_index, - QualifiedIdTypeJoinableIndex::Create( - filesystem, GetQualifiedIdJoinIndexDir(), /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/false)); + std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index, + QualifiedIdJoinIndex::Create(filesystem, GetQualifiedIdJoinIndexDir(), + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false)); EXPECT_THAT(qualified_id_join_index->Get( DocJoinInfo(/*document_id=*/0, /*joinable_property_id=*/0)), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); @@ -3742,10 +3866,10 @@ TEST_F(IcingSearchEngineInitializationTest, { Filesystem filesystem; ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> qualified_id_join_index, - QualifiedIdTypeJoinableIndex::Create( - filesystem, GetQualifiedIdJoinIndexDir(), /*pre_mapping_fbv=*/false, - /*use_persistent_hash_map=*/false)); + std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index, + QualifiedIdJoinIndex::Create(filesystem, GetQualifiedIdJoinIndexDir(), + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false)); // Add data for document 4. DocumentId original_last_added_doc_id = qualified_id_join_index->last_added_document_id(); @@ -3839,10 +3963,9 @@ TEST_F(IcingSearchEngineInitializationTest, // `name:person` with a child query for `body:consectetur` based on the // child's `senderQualifiedId` field. - // Add document 4 without "senderQualifiedId". If joinable index is not - // rebuilt correctly, then it will still have the previously added - // senderQualifiedId for document 4 and include document 4 incorrectly in - // the right side. + // Add document 4 without "senderQualifiedId". If join index is not rebuilt + // correctly, then it will still have the previously added senderQualifiedId + // for document 4 and include document 4 incorrectly in the right side. DocumentProto another_message = DocumentBuilder() .SetKey("namespace", "message/4") @@ -4307,7 +4430,9 @@ TEST_F(IcingSearchEngineInitializationTest, ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Index> index, Index::Create(Index::Options(GetIndexDir(), - /*index_merge_size=*/100), + /*index_merge_size=*/100, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/50), filesystem(), icing_filesystem())); ICING_ASSERT_OK(index->PersistToDisk()); } @@ -4829,7 +4954,7 @@ TEST_F(IcingSearchEngineInitializationTest, auto mock_filesystem = std::make_unique<MockFilesystem>(); EXPECT_CALL(*mock_filesystem, PRead(A<const char*>(), _, _, _)) .WillRepeatedly(DoDefault()); - // This fails QualifiedIdTypeJoinableIndex::Create() once. + // This fails QualifiedIdJoinIndex::Create() once. EXPECT_CALL( *mock_filesystem, PRead(Matcher<const char*>(Eq(qualified_id_join_index_metadata_file)), _, @@ -5123,19 +5248,22 @@ TEST_P(IcingSearchEngineInitializationVersionChangeTest, // Put message into DocumentStore ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(filesystem(), GetDocumentDir(), &fake_clock, - schema_store.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + filesystem(), GetDocumentDir(), &fake_clock, schema_store.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); std::unique_ptr<DocumentStore> document_store = std::move(create_result.document_store); ICING_ASSERT_OK_AND_ASSIGN(DocumentId doc_id, document_store->Put(message)); // Index doc_id with incorrect data - Index::Options options(GetIndexDir(), /*index_merge_size=*/1024 * 1024); + Index::Options options(GetIndexDir(), /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<Index> index, Index::Create(options, filesystem(), icing_filesystem())); @@ -5143,11 +5271,12 @@ TEST_P(IcingSearchEngineInitializationVersionChangeTest, ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(*filesystem(), GetIntegerIndexDir(), + /*num_data_threshold_for_bucket_split=*/65536, /*pre_mapping_fbv=*/false)); ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> qualified_id_join_index, - QualifiedIdTypeJoinableIndex::Create( + std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index, + QualifiedIdJoinIndex::Create( *filesystem(), GetQualifiedIdJoinIndexDir(), /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/false)); @@ -5160,16 +5289,14 @@ TEST_P(IcingSearchEngineInitializationVersionChangeTest, integer_section_indexing_handler, IntegerSectionIndexingHandler::Create( &fake_clock, integer_index.get())); - ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdJoinIndexingHandler> - qualified_id_joinable_property_indexing_handler, - QualifiedIdJoinIndexingHandler::Create(&fake_clock, - qualified_id_join_index.get())); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexingHandler> + qualified_id_join_indexing_handler, + QualifiedIdJoinIndexingHandler::Create( + &fake_clock, qualified_id_join_index.get())); std::vector<std::unique_ptr<DataIndexingHandler>> handlers; handlers.push_back(std::move(string_section_indexing_handler)); handlers.push_back(std::move(integer_section_indexing_handler)); - handlers.push_back( - std::move(qualified_id_joinable_property_indexing_handler)); + handlers.push_back(std::move(qualified_id_join_indexing_handler)); IndexProcessor index_processor(std::move(handlers), &fake_clock); DocumentProto incorrect_message = @@ -5302,12 +5429,8 @@ INSTANTIATE_TEST_SUITE_P( /*version_in=*/version_util::kVersion + 1, /*max_version_in=*/version_util::kVersion + 1), - // Manually change existing data set's version to kVersion - 1 and - // max_version to kVersion - 1. When initializing, it will detect - // "upgrade". - version_util::VersionInfo( - /*version_in=*/version_util::kVersion - 1, - /*max_version_in=*/version_util::kVersion - 1), + // Currently we don't have any "upgrade" that requires rebuild derived + // files, so skip this case until we have a case for it. // Manually change existing data set's version to kVersion - 1 and // max_version to kVersion. When initializing, it will detect "roll diff --git a/icing/icing-search-engine_schema_test.cc b/icing/icing-search-engine_schema_test.cc index 2609cce..d665673 100644 --- a/icing/icing-search-engine_schema_test.cc +++ b/icing/icing-search-engine_schema_test.cc @@ -18,7 +18,6 @@ #include <string> #include <utility> -#include "icing/text_classifier/lib3/utils/base/status.h" #include "gmock/gmock.h" #include "gtest/gtest.h" #include "icing/document-builder.h" @@ -27,14 +26,12 @@ #include "icing/icing-search-engine.h" #include "icing/jni/jni-cache.h" #include "icing/join/join-processor.h" -#include "icing/portable/endian.h" #include "icing/portable/equals-proto.h" #include "icing/portable/platform.h" #include "icing/proto/debug.pb.h" #include "icing/proto/document.pb.h" #include "icing/proto/document_wrapper.pb.h" #include "icing/proto/initialize.pb.h" -#include "icing/proto/logging.pb.h" #include "icing/proto/optimize.pb.h" #include "icing/proto/persist.pb.h" #include "icing/proto/reset.pb.h" @@ -888,6 +885,256 @@ TEST_F(IcingSearchEngineSchemaTest, expected_search_result_proto)); } +TEST_F( + IcingSearchEngineSchemaTest, + SetSchemaNewIndexedDocumentPropertyTriggersIndexRestorationAndReturnsOk) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + // Create a schema with a nested document type: + // + // Section id assignment for 'Person': + // - "age": integer type, indexed. Section id = 0 + // - "name": string type, indexed. Section id = 1. + // - "worksFor.name": string type, (nested) indexed. Section id = 2. + // + // Joinable property id assignment for 'Person': + // - "worksFor.listRef": string type, Qualified Id type joinable. Joinable + // property id = 0. + SchemaProto schema_one = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("List").AddProperty( + PropertyConfigBuilder() + .SetName("title") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty(PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("age") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("worksFor") + .SetDataTypeDocument( + "Organization", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Organization") + .AddProperty(PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("listRef") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + ASSERT_THAT(icing.SetSchema(schema_one).status(), ProtoIsOk()); + + DocumentProto list_document = DocumentBuilder() + .SetKey("namespace", "list/1") + .SetSchema("List") + .SetCreationTimestampMs(1000) + .AddStringProperty("title", "title") + .Build(); + DocumentProto person_document = + DocumentBuilder() + .SetKey("namespace", "person/2") + .SetSchema("Person") + .SetCreationTimestampMs(1000) + .AddStringProperty("name", "John") + .AddInt64Property("age", 20) + .AddDocumentProperty( + "worksFor", DocumentBuilder() + .SetKey("namespace", "org/1") + .SetSchema("Organization") + .AddStringProperty("name", "Google") + .AddStringProperty("listRef", "namespace#list/1") + .Build()) + .Build(); + EXPECT_THAT(icing.Put(list_document).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(person_document).status(), ProtoIsOk()); + + ResultSpecProto result_spec = ResultSpecProto::default_instance(); + result_spec.set_max_joined_children_per_parent_to_return( + std::numeric_limits<int32_t>::max()); + + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code(StatusProto::OK); + *expected_search_result_proto.mutable_results()->Add()->mutable_document() = + person_document; + + SearchResultProto empty_result; + empty_result.mutable_status()->set_code(StatusProto::OK); + + // Verify term search + SearchSpecProto search_spec1; + search_spec1.set_query("worksFor.name:Google"); + search_spec1.set_term_match_type(TermMatchType::EXACT_ONLY); + + SearchResultProto actual_results = + icing.Search(search_spec1, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); + + // Verify numeric (integer) search + SearchSpecProto search_spec2; + search_spec2.set_query("age == 20"); + search_spec2.set_search_type( + SearchSpecProto::SearchType::EXPERIMENTAL_ICING_ADVANCED_QUERY); + search_spec2.add_enabled_features(std::string(kNumericSearchFeature)); + + actual_results = + icing.Search(search_spec2, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); + + // Verify join search: join a query for `title:title` (which will get + // list_document) with a child query for `name:John` (which will get + // person_document) based on the child's `worksFor.listRef` field. + SearchSpecProto search_spec_with_join; + search_spec_with_join.set_query("title:title"); + search_spec_with_join.set_term_match_type(TermMatchType::EXACT_ONLY); + JoinSpecProto* join_spec = search_spec_with_join.mutable_join_spec(); + join_spec->set_parent_property_expression( + std::string(JoinProcessor::kQualifiedIdExpr)); + join_spec->set_child_property_expression("worksFor.listRef"); + join_spec->set_aggregation_scoring_strategy( + JoinSpecProto::AggregationScoringStrategy::COUNT); + JoinSpecProto::NestedSpecProto* nested_spec = + join_spec->mutable_nested_spec(); + SearchSpecProto* nested_search_spec = nested_spec->mutable_search_spec(); + nested_search_spec->set_term_match_type(TermMatchType::EXACT_ONLY); + nested_search_spec->set_query("name:John"); + *nested_spec->mutable_scoring_spec() = GetDefaultScoringSpec(); + *nested_spec->mutable_result_spec() = result_spec; + + SearchResultProto expected_join_search_result_proto; + expected_join_search_result_proto.mutable_status()->set_code(StatusProto::OK); + SearchResultProto::ResultProto* result_proto = + expected_join_search_result_proto.mutable_results()->Add(); + *result_proto->mutable_document() = list_document; + *result_proto->mutable_joined_results()->Add()->mutable_document() = + person_document; + + actual_results = + icing.Search(search_spec_with_join, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_join_search_result_proto)); + + // Change the schema to add another nested document property to 'Person' + // + // New section id assignment for 'Person': + // - "age": integer type, indexed. Section id = 0 + // - "almaMater.name", string type, indexed. Section id = 1 + // - "name": string type, indexed. Section id = 2 + // - "worksFor.name": string type, (nested) indexed. Section id = 3 + // + // New joinable property id assignment for 'Person': + // - "almaMater.listRef": string type, Qualified Id type joinable. Joinable + // property id = 0. + // - "worksFor.listRef": string type, Qualified Id type joinable. Joinable + // property id = 1. + SchemaProto schema_two = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("List").AddProperty( + PropertyConfigBuilder() + .SetName("title") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty(PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("age") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("worksFor") + .SetDataTypeDocument( + "Organization", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("almaMater") + .SetDataTypeDocument( + "Organization", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Organization") + .AddProperty(PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REQUIRED)) + .AddProperty(PropertyConfigBuilder() + .SetName("listRef") + .SetDataTypeJoinableString( + JOINABLE_VALUE_TYPE_QUALIFIED_ID) + .SetCardinality(CARDINALITY_REQUIRED))) + .Build(); + + // This schema change is compatible since the added 'almaMater' property has + // CARDINALITY_OPTIONAL. + // + // Index restoration should be triggered here because new schema requires more + // properties to be indexed. Also new section ids will be reassigned and index + // restoration should use new section ids to rebuild. + SetSchemaResultProto set_schema_result = icing.SetSchema(schema_two); + // Ignore latency numbers. They're covered elsewhere. + set_schema_result.clear_latency_ms(); + SetSchemaResultProto expected_set_schema_result = SetSchemaResultProto(); + expected_set_schema_result.mutable_status()->set_code(StatusProto::OK); + expected_set_schema_result.mutable_index_incompatible_changed_schema_types() + ->Add("Person"); + expected_set_schema_result.mutable_join_incompatible_changed_schema_types() + ->Add("Person"); + EXPECT_THAT(set_schema_result, EqualsProto(expected_set_schema_result)); + + // Verify term search: + // Searching for "worksFor.name:Google" should still match document + actual_results = + icing.Search(search_spec1, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); + + // In new_schema the 'name' property is now indexed at section id 2. If + // searching for "name:Google" matched the document, this means that index + // rebuild was not triggered and Icing is still searching the old index, where + // 'worksFor.name' was indexed at section id 2. + search_spec1.set_query("name:Google"); + actual_results = + icing.Search(search_spec1, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(actual_results, + EqualsSearchResultIgnoreStatsAndScores(empty_result)); + + // Verify numeric (integer) search: should still match document + actual_results = + icing.Search(search_spec2, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); + + // Verify join search: should still able to join by `worksFor.listRef` + actual_results = + icing.Search(search_spec_with_join, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_join_search_result_proto)); +} + TEST_F(IcingSearchEngineSchemaTest, SetSchemaChangeNestedPropertiesTriggersIndexRestorationAndReturnsOk) { IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); @@ -1081,6 +1328,281 @@ TEST_F(IcingSearchEngineSchemaTest, EqualsSearchResultIgnoreStatsAndScores(empty_result)); } +TEST_F( + IcingSearchEngineSchemaTest, + SetSchemaChangeNestedPropertiesListTriggersIndexRestorationAndReturnsOk) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + SchemaTypeConfigProto person_proto = + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("lastName") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("address") + .SetDataTypeString(TERM_MATCH_UNKNOWN, TOKENIZER_NONE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("age") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("birthday") + .SetDataTypeInt64(NUMERIC_MATCH_UNKNOWN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + // Create a schema with nested properties: + // - "sender.address": string type, (nested) non-indexable. Section id = 0. + // - "sender.age": int64 type, (nested) indexed. Section id = 1. + // - "sender.birthday": int64 type, (nested) non-indexable. Section id = 2. + // - "sender.lastName": int64 type, (nested) indexed. Section id = 3. + // - "sender.name": string type, (nested) indexed. Section id = 4. + // - "subject": string type, indexed. Section id = 5. + // - "timestamp": int64 type, indexed. Section id = 6. + // - "sender.foo": unknown type, (nested) non-indexable. Section id = 7. + // + // "sender.address" and "sender.birthday" are assigned a section id because + // they are listed in the indexable_nested_properties_list for 'Email.sender'. + // They are assigned a sectionId but are not indexed since their indexing + // configs are non-indexable. + // + // "sender.foo" is also assigned a section id, but is also not undefined by + // the schema definition. Trying to index a document with this nested property + // should fail. + SchemaProto nested_schema = + SchemaBuilder() + .AddType(person_proto) + .AddType( + SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty( + PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeDocument( + "Person", /*indexable_nested_properties_list=*/ + {"age", "lastName", "address", "name", "birthday", + "foo"}) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("timestamp") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + SetSchemaResultProto set_schema_result = icing.SetSchema(nested_schema); + // Ignore latency numbers. They're covered elsewhere. + set_schema_result.clear_latency_ms(); + SetSchemaResultProto expected_set_schema_result; + expected_set_schema_result.mutable_status()->set_code(StatusProto::OK); + expected_set_schema_result.mutable_new_schema_types()->Add("Email"); + expected_set_schema_result.mutable_new_schema_types()->Add("Person"); + EXPECT_THAT(set_schema_result, EqualsProto(expected_set_schema_result)); + + DocumentProto document = + DocumentBuilder() + .SetKey("namespace1", "uri1") + .SetSchema("Email") + .SetCreationTimestampMs(1000) + .AddStringProperty("subject", + "Did you get the memo about TPS reports?") + .AddDocumentProperty( + "sender", + DocumentBuilder() + .SetKey("namespace1", "uri1") + .SetSchema("Person") + .AddStringProperty("name", "Bill") + .AddStringProperty("lastName", "Lundbergh") + .AddStringProperty("address", "1600 Amphitheatre Pkwy") + .AddInt64Property("age", 20) + .AddInt64Property("birthday", 20) + .Build()) + .AddInt64Property("timestamp", 1234) + .Build(); + + // Indexing this doc should fail, since the 'sender.foo' property is not found + DocumentProto invalid_document = + DocumentBuilder() + .SetKey("namespace2", "uri1") + .SetSchema("Email") + .SetCreationTimestampMs(1000) + .AddStringProperty("subject", + "Did you get the memo about TPS reports?") + .AddDocumentProperty( + "sender", + DocumentBuilder() + .SetKey("namespace1", "uri1") + .SetSchema("Person") + .AddStringProperty("name", "Bill") + .AddStringProperty("lastName", "Lundbergh") + .AddStringProperty("address", "1600 Amphitheatre Pkwy") + .AddInt64Property("age", 20) + .AddInt64Property("birthday", 20) + .AddBytesProperty("foo", "bar bytes") + .Build()) + .AddInt64Property("timestamp", 1234) + .Build(); + + EXPECT_THAT(icing.Put(document).status(), ProtoIsOk()); + EXPECT_THAT(icing.Put(invalid_document).status(), + ProtoStatusIs(StatusProto::NOT_FOUND)); + + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code(StatusProto::OK); + *expected_search_result_proto.mutable_results()->Add()->mutable_document() = + document; + + SearchResultProto empty_result; + empty_result.mutable_status()->set_code(StatusProto::OK); + + // Verify term search + // document should match a query for 'Bill' in 'sender.name', but not in + // 'sender.lastName' + SearchSpecProto search_spec1; + search_spec1.set_query("sender.name:Bill"); + search_spec1.set_term_match_type(TermMatchType::EXACT_ONLY); + + SearchResultProto actual_results = + icing.Search(search_spec1, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); + + search_spec1.set_query("sender.lastName:Bill"); + actual_results = icing.Search(search_spec1, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, + EqualsSearchResultIgnoreStatsAndScores(empty_result)); + + // document should match a query for 'Lundber' in 'sender.lastName', but not + // in 'sender.name'. + SearchSpecProto search_spec2; + search_spec2.set_query("sender.lastName:Lundber"); + search_spec2.set_term_match_type(TermMatchType::PREFIX); + + actual_results = icing.Search(search_spec2, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); + + search_spec2.set_query("sender.name:Lundber"); + actual_results = icing.Search(search_spec2, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, + EqualsSearchResultIgnoreStatsAndScores(empty_result)); + + // document should not match a query for 'Amphitheatre' because the + // 'sender.address' field is not indexed. + search_spec2.set_query("Amphitheatre"); + search_spec2.set_term_match_type(TermMatchType::PREFIX); + + actual_results = icing.Search(search_spec2, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, + EqualsSearchResultIgnoreStatsAndScores(empty_result)); + + // Verify numeric (integer) search + // document should match a query for 20 in 'sender.age', but not in + // 'timestamp' or 'sender.birthday' + SearchSpecProto search_spec3; + search_spec3.set_query("sender.age == 20"); + search_spec3.set_search_type( + SearchSpecProto::SearchType::EXPERIMENTAL_ICING_ADVANCED_QUERY); + search_spec3.add_enabled_features(std::string(kNumericSearchFeature)); + + actual_results = icing.Search(search_spec3, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); + + search_spec3.set_query("timestamp == 20"); + actual_results = icing.Search(search_spec3, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, + EqualsSearchResultIgnoreStatsAndScores(empty_result)); + + search_spec3.set_query("birthday == 20"); + actual_results = icing.Search(search_spec3, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, + EqualsSearchResultIgnoreStatsAndScores(empty_result)); + + // Now update the schema and don't index "sender.name", "sender.birthday" and + // "sender.foo". + // This should reassign section ids, lead to an index rebuild and ensure that + // nothing match a query for "Bill". + // + // Section id assignment: + // - "sender.address": string type, (nested) non-indexable. Section id = 0. + // - "sender.age": int64 type, (nested) indexed. Section id = 1. + // - "sender.birthday": int64 type, (nested) unindexed. No section id. + // - "sender.lastName": int64 type, (nested) indexed. Section id = 2. + // - "sender.name": string type, (nested) unindexed. No section id. + // - "subject": string type, indexed. Section id = 3. + // - "timestamp": int64 type, indexed. Section id = 4. + // - "sender.foo": unknown type, invalid. No section id. + SchemaProto nested_schema_with_less_props = + SchemaBuilder() + .AddType(person_proto) + .AddType(SchemaTypeConfigBuilder() + .SetType("Email") + .AddProperty( + PropertyConfigBuilder() + .SetName("sender") + .SetDataTypeDocument( + "Person", /*indexable_nested_properties=*/ + {"age", "lastName", "address"}) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("timestamp") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + set_schema_result = icing.SetSchema(nested_schema_with_less_props); + // Ignore latency numbers. They're covered elsewhere. + set_schema_result.clear_latency_ms(); + expected_set_schema_result = SetSchemaResultProto(); + expected_set_schema_result.mutable_status()->set_code(StatusProto::OK); + expected_set_schema_result.mutable_index_incompatible_changed_schema_types() + ->Add("Email"); + EXPECT_THAT(set_schema_result, EqualsProto(expected_set_schema_result)); + + // Verify term search + // document shouldn't match a query for 'Bill' in either 'sender.name' or + // 'subject' + search_spec1.set_query("sender.name:Bill"); + actual_results = icing.Search(search_spec1, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, + EqualsSearchResultIgnoreStatsAndScores(empty_result)); + + search_spec1.set_query("subject:Bill"); + actual_results = icing.Search(search_spec1, GetDefaultScoringSpec(), + ResultSpecProto::default_instance()); + EXPECT_THAT(actual_results, + EqualsSearchResultIgnoreStatsAndScores(empty_result)); +} + TEST_F(IcingSearchEngineSchemaTest, SetSchemaNewJoinablePropertyTriggersIndexRestorationAndReturnsOk) { IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); @@ -1614,8 +2136,8 @@ TEST_F(IcingSearchEngineSchemaTest, // - "senderQualifiedId": qualified id joinable. Joinable property id = 0. // // If the index is not correctly rebuilt, then the joinable data of - // "senderQualifiedId" in the joinable index will still have old joinable - // property id of 1 and therefore won't take effect for join search query. + // "senderQualifiedId" in the join index will still have old joinable property + // id of 1 and therefore won't take effect for join search query. SchemaProto email_without_receiver_schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( @@ -1917,8 +2439,8 @@ TEST_F( // - "zQualifiedId": qualified id joinable. Joinable property id = 1. // // If the index is not correctly rebuilt, then the joinable data of - // "senderQualifiedId" in the joinable index will still have old joinable - // property id of 1 and therefore won't take effect for join search query. + // "senderQualifiedId" in the join index will still have old joinable property + // id of 1 and therefore won't take effect for join search query. SchemaProto email_no_body_schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType("Person").AddProperty( @@ -2609,6 +3131,26 @@ TEST_F(IcingSearchEngineSchemaTest, IcingShouldWorkFor64Sections) { EqualsSearchResultIgnoreStatsAndScores(expected_no_documents)); } +TEST_F(IcingSearchEngineSchemaTest, IcingShouldReturnErrorForExtraSections) { + // Create a schema with more sections than allowed. + SchemaTypeConfigBuilder schema_type_config_builder = + SchemaTypeConfigBuilder().SetType("type"); + for (int i = 0; i <= kMaxSectionId + 1; ++i) { + schema_type_config_builder.AddProperty( + PropertyConfigBuilder() + .SetName("prop" + std::to_string(i)) + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)); + } + SchemaProto schema = + SchemaBuilder().AddType(schema_type_config_builder).Build(); + + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(schema).status().message(), + HasSubstr("Too many properties to be indexed")); +} + } // namespace } // namespace lib } // namespace icing diff --git a/icing/icing-search-engine_search_test.cc b/icing/icing-search-engine_search_test.cc index f1b49fb..451c9ce 100644 --- a/icing/icing-search-engine_search_test.cc +++ b/icing/icing-search-engine_search_test.cc @@ -502,6 +502,71 @@ TEST_P(IcingSearchEngineSearchTest, expected_search_result_proto)); } +TEST_P(IcingSearchEngineSearchTest, SearchWithNumToScore) { + auto fake_clock = std::make_unique<FakeClock>(); + fake_clock->SetTimerElapsedMilliseconds(1000); + TestIcingSearchEngine icing(GetDefaultIcingOptions(), + std::make_unique<Filesystem>(), + std::make_unique<IcingFilesystem>(), + std::move(fake_clock), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk()); + + DocumentProto document_one = CreateMessageDocument("namespace", "uri1"); + document_one.set_score(10); + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + + DocumentProto document_two = CreateMessageDocument("namespace", "uri2"); + document_two.set_score(5); + ASSERT_THAT(icing.Put(document_two).status(), ProtoIsOk()); + + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::PREFIX); + search_spec.set_query("message"); + search_spec.set_search_type(GetParam()); + + ResultSpecProto result_spec; + result_spec.set_num_per_page(10); + result_spec.set_num_to_score(10); + + ScoringSpecProto scoring_spec = GetDefaultScoringSpec(); + + SearchResultProto expected_search_result_proto1; + expected_search_result_proto1.mutable_status()->set_code(StatusProto::OK); + *expected_search_result_proto1.mutable_results()->Add()->mutable_document() = + document_one; + *expected_search_result_proto1.mutable_results()->Add()->mutable_document() = + document_two; + + SearchResultProto search_result_proto = + icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto1)); + + result_spec.set_num_to_score(1); + // By setting num_to_score = 1, only document_two will be scored, ranked, and + // returned. + // - num_to_score cutoff is only affected by the reading order from posting + // list. IOW, since we read posting lists in doc id descending order, + // ScoringProcessor scores documents with higher doc ids first and cuts off + // if exceeding num_to_score. + // - Therefore, even though document_one has higher score, ScoringProcessor + // still skips document_one, because posting list reads document_two first + // and ScoringProcessor stops after document_two given that total # of + // scored document has already reached num_to_score. + SearchResultProto expected_search_result_google::protobuf; + expected_search_result_google::protobuf.mutable_status()->set_code(StatusProto::OK); + *expected_search_result_google::protobuf.mutable_results()->Add()->mutable_document() = + document_two; + + search_result_proto = + icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(search_result_proto.status(), ProtoIsOk()); + EXPECT_THAT(search_result_proto, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_google::protobuf)); +} + TEST_P(IcingSearchEngineSearchTest, SearchNegativeResultLimitReturnsInvalidArgument) { IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); @@ -577,6 +642,62 @@ TEST_P(IcingSearchEngineSearchTest, ProtoStatusIs(StatusProto::INVALID_ARGUMENT)); } +TEST_P(IcingSearchEngineSearchTest, + SearchNegativeMaxJoinedChildrenPerParentReturnsInvalidArgument) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::PREFIX); + search_spec.set_query(""); + search_spec.set_search_type(GetParam()); + + ResultSpecProto result_spec; + result_spec.set_max_joined_children_per_parent_to_return(-1); + + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code( + StatusProto::INVALID_ARGUMENT); + expected_search_result_proto.mutable_status()->set_message( + "ResultSpecProto.max_joined_children_per_parent_to_return cannot be " + "negative."); + SearchResultProto actual_results = + icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(actual_results, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); +} + +TEST_P(IcingSearchEngineSearchTest, + SearchNonPositiveNumToScoreReturnsInvalidArgument) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + + SearchSpecProto search_spec; + search_spec.set_term_match_type(TermMatchType::PREFIX); + search_spec.set_query(""); + search_spec.set_search_type(GetParam()); + + ResultSpecProto result_spec; + result_spec.set_num_to_score(-1); + + SearchResultProto expected_search_result_proto; + expected_search_result_proto.mutable_status()->set_code( + StatusProto::INVALID_ARGUMENT); + expected_search_result_proto.mutable_status()->set_message( + "ResultSpecProto.num_to_score cannot be non-positive."); + + SearchResultProto actual_results1 = + icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(actual_results1, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); + + result_spec.set_num_to_score(0); + SearchResultProto actual_results2 = + icing.Search(search_spec, GetDefaultScoringSpec(), result_spec); + EXPECT_THAT(actual_results2, EqualsSearchResultIgnoreStatsAndScores( + expected_search_result_proto)); +} + TEST_P(IcingSearchEngineSearchTest, SearchWithPersistenceReturnsValidResults) { IcingSearchEngineOptions icing_options = GetDefaultIcingOptions(); @@ -3444,6 +3565,547 @@ TEST_P(IcingSearchEngineSearchTest, SearchWithProjectionMultipleFieldPaths) { EqualsProto(projected_document_one)); } +TEST_P(IcingSearchEngineSearchTest, SearchWithPropertyFilters) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreatePersonAndEmailSchema()).status(), + ProtoIsOk()); + + // 1. Add two email documents + DocumentProto document_one = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetCreationTimestampMs(1000) + .SetSchema("Email") + .AddDocumentProperty( + "sender", + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("Person") + .AddStringProperty("name", "Meg Ryan") + .AddStringProperty("emailAddress", "hellogirl@aol.com") + .Build()) + .AddStringProperty("subject", "Hello World!") + .AddStringProperty( + "body", "Oh what a beautiful morning! Oh what a beautiful day!") + .Build(); + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + + DocumentProto document_two = + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetCreationTimestampMs(1000) + .SetSchema("Email") + .AddDocumentProperty( + "sender", DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("Person") + .AddStringProperty("name", "Tom Hanks") + .AddStringProperty("emailAddress", "ny152@aol.com") + .Build()) + .AddStringProperty("subject", "Goodnight Moon!") + .AddStringProperty("body", + "Count all the sheep and tell them 'Hello'.") + .Build(); + ASSERT_THAT(icing.Put(document_two).status(), ProtoIsOk()); + + // 2. Issue a query with property filters of sender.name and subject for the + // Email schema type. + auto search_spec = std::make_unique<SearchSpecProto>(); + search_spec->set_term_match_type(TermMatchType::PREFIX); + search_spec->set_query("hello"); + search_spec->set_search_type(GetParam()); + TypePropertyMask* email_property_filters = + search_spec->add_type_property_filters(); + email_property_filters->set_schema_type("Email"); + email_property_filters->add_paths("sender.name"); + email_property_filters->add_paths("subject"); + + auto result_spec = std::make_unique<ResultSpecProto>(); + + auto scoring_spec = std::make_unique<ScoringSpecProto>(); + *scoring_spec = GetDefaultScoringSpec(); + SearchResultProto results = + icing.Search(*search_spec, *scoring_spec, *result_spec); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(1)); + + // 3. Verify that only the first document is returned. Although 'hello' is + // present in document_two, it shouldn't be in the result since 'hello' is not + // in the specified property filter. + EXPECT_THAT(results.results(0).document(), + EqualsProto(document_one)); +} + +TEST_P(IcingSearchEngineSearchTest, SearchWithPropertyFiltersOnMultipleSchema) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + // Add Person and Organization schema with a property 'name' in both. + SchemaProto schema = SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty(PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("emailAddress") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("Organization") + .AddProperty(PropertyConfigBuilder() + .SetName("name") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("address") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + ASSERT_THAT(icing.SetSchema(schema).status(), + ProtoIsOk()); + + // 1. Add person document + DocumentProto person_document = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetCreationTimestampMs(1000) + .SetSchema("Person") + .AddStringProperty("name", "Meg Ryan") + .AddStringProperty("emailAddress", "hellogirl@aol.com") + .Build(); + ASSERT_THAT(icing.Put(person_document).status(), ProtoIsOk()); + + // 1. Add organization document + DocumentProto organization_document = + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetCreationTimestampMs(1000) + .SetSchema("Organization") + .AddStringProperty("name", "Meg Corp") + .AddStringProperty("address", "Universal street") + .Build(); + ASSERT_THAT(icing.Put(organization_document).status(), ProtoIsOk()); + + // 2. Issue a query with property filters. Person schema has name in it's + // property filter but Organization schema doesn't. + auto search_spec = std::make_unique<SearchSpecProto>(); + search_spec->set_term_match_type(TermMatchType::PREFIX); + search_spec->set_query("Meg"); + search_spec->set_search_type(GetParam()); + TypePropertyMask* person_property_filters = + search_spec->add_type_property_filters(); + person_property_filters->set_schema_type("Person"); + person_property_filters->add_paths("name"); + TypePropertyMask* organization_property_filters = + search_spec->add_type_property_filters(); + organization_property_filters->set_schema_type("Organization"); + organization_property_filters->add_paths("address"); + + auto result_spec = std::make_unique<ResultSpecProto>(); + + auto scoring_spec = std::make_unique<ScoringSpecProto>(); + *scoring_spec = GetDefaultScoringSpec(); + SearchResultProto results = + icing.Search(*search_spec, *scoring_spec, *result_spec); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(1)); + + // 3. Verify that only the person document is returned. Although 'Meg' is + // present in organization document, it shouldn't be in the result since + // the name field is not specified in the Organization property filter. + EXPECT_THAT(results.results(0).document(), + EqualsProto(person_document)); +} + +TEST_P(IcingSearchEngineSearchTest, SearchWithWildcardPropertyFilters) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreatePersonAndEmailSchema()).status(), + ProtoIsOk()); + + // 1. Add two email documents + DocumentProto document_one = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetCreationTimestampMs(1000) + .SetSchema("Email") + .AddDocumentProperty( + "sender", + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("Person") + .AddStringProperty("name", "Meg Ryan") + .AddStringProperty("emailAddress", "hellogirl@aol.com") + .Build()) + .AddStringProperty("subject", "Hello World!") + .AddStringProperty( + "body", "Oh what a beautiful morning! Oh what a beautiful day!") + .Build(); + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + + DocumentProto document_two = + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetCreationTimestampMs(1000) + .SetSchema("Email") + .AddDocumentProperty( + "sender", DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("Person") + .AddStringProperty("name", "Tom Hanks") + .AddStringProperty("emailAddress", "ny152@aol.com") + .Build()) + .AddStringProperty("subject", "Goodnight Moon!") + .AddStringProperty("body", + "Count all the sheep and tell them 'Hello'.") + .Build(); + ASSERT_THAT(icing.Put(document_two).status(), ProtoIsOk()); + + // 2. Issue a query with property filters of sender.name and subject for the + // wildcard(*) schema type. + auto search_spec = std::make_unique<SearchSpecProto>(); + search_spec->set_term_match_type(TermMatchType::PREFIX); + search_spec->set_query("hello"); + search_spec->set_search_type(GetParam()); + TypePropertyMask* wildcard_property_filters = + search_spec->add_type_property_filters(); + wildcard_property_filters->set_schema_type("*"); + wildcard_property_filters->add_paths("sender.name"); + wildcard_property_filters->add_paths("subject"); + + auto result_spec = std::make_unique<ResultSpecProto>(); + + auto scoring_spec = std::make_unique<ScoringSpecProto>(); + *scoring_spec = GetDefaultScoringSpec(); + SearchResultProto results = + icing.Search(*search_spec, *scoring_spec, *result_spec); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(1)); + + // 3. Verify that only the first document is returned since the second + // document doesn't contain the word 'hello' in either of fields specified in + // the property filter. This confirms that the property filters for the + // wildcard entry have been applied to the Email schema as well. + EXPECT_THAT(results.results(0).document(), + EqualsProto(document_one)); +} + +TEST_P(IcingSearchEngineSearchTest, SearchWithMixedPropertyFilters) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreatePersonAndEmailSchema()).status(), + ProtoIsOk()); + + // 1. Add two email documents + DocumentProto document_one = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetCreationTimestampMs(1000) + .SetSchema("Email") + .AddDocumentProperty( + "sender", + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("Person") + .AddStringProperty("name", "Meg Ryan") + .AddStringProperty("emailAddress", "hellogirl@aol.com") + .Build()) + .AddStringProperty("subject", "Hello World!") + .AddStringProperty( + "body", "Oh what a beautiful morning! Oh what a beautiful day!") + .Build(); + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + + DocumentProto document_two = + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetCreationTimestampMs(1000) + .SetSchema("Email") + .AddDocumentProperty( + "sender", DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("Person") + .AddStringProperty("name", "Tom Hanks") + .AddStringProperty("emailAddress", "ny152@aol.com") + .Build()) + .AddStringProperty("subject", "Goodnight Moon!") + .AddStringProperty("body", + "Count all the sheep and tell them 'Hello'.") + .Build(); + ASSERT_THAT(icing.Put(document_two).status(), ProtoIsOk()); + + // 2. Issue a query with property filters of sender.name and subject for the + // wildcard(*) schema type plus property filters of sender.name and body for + // the Email schema type. + auto search_spec = std::make_unique<SearchSpecProto>(); + search_spec->set_term_match_type(TermMatchType::PREFIX); + search_spec->set_query("hello"); + search_spec->set_search_type(GetParam()); + TypePropertyMask* wildcard_property_filters = + search_spec->add_type_property_filters(); + wildcard_property_filters->set_schema_type("*"); + wildcard_property_filters->add_paths("sender.name"); + wildcard_property_filters->add_paths("subject"); + TypePropertyMask* email_property_filters = + search_spec->add_type_property_filters(); + email_property_filters->set_schema_type("Email"); + email_property_filters->add_paths("sender.name"); + email_property_filters->add_paths("body"); + + auto result_spec = std::make_unique<ResultSpecProto>(); + + auto scoring_spec = std::make_unique<ScoringSpecProto>(); + *scoring_spec = GetDefaultScoringSpec(); + SearchResultProto results = + icing.Search(*search_spec, *scoring_spec, *result_spec); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(1)); + + // 3. Verify that only the second document is returned since the first + // document doesn't contain the word 'hello' in either of fields sender.name + // or body. This confirms that the property filters specified for Email schema + // have been applied and the ones specified for wildcard entry have been + // ignored. + EXPECT_THAT(results.results(0).document(), + EqualsProto(document_two)); +} + +TEST_P(IcingSearchEngineSearchTest, SearchWithNonApplicablePropertyFilters) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreatePersonAndEmailSchema()).status(), + ProtoIsOk()); + + // 1. Add two email documents + DocumentProto document_one = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetCreationTimestampMs(1000) + .SetSchema("Email") + .AddDocumentProperty( + "sender", + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("Person") + .AddStringProperty("name", "Meg Ryan") + .AddStringProperty("emailAddress", "hellogirl@aol.com") + .Build()) + .AddStringProperty("subject", "Hello World!") + .AddStringProperty( + "body", "Oh what a beautiful morning! Oh what a beautiful day!") + .Build(); + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + + DocumentProto document_two = + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetCreationTimestampMs(1000) + .SetSchema("Email") + .AddDocumentProperty( + "sender", DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("Person") + .AddStringProperty("name", "Tom Hanks") + .AddStringProperty("emailAddress", "ny152@aol.com") + .Build()) + .AddStringProperty("subject", "Goodnight Moon!") + .AddStringProperty("body", + "Count all the sheep and tell them 'Hello'.") + .Build(); + ASSERT_THAT(icing.Put(document_two).status(), ProtoIsOk()); + + // 2. Issue a query with property filters of sender.name and subject for an + // unknown schema type. + auto search_spec = std::make_unique<SearchSpecProto>(); + search_spec->set_term_match_type(TermMatchType::PREFIX); + search_spec->set_query("hello"); + search_spec->set_search_type(GetParam()); + TypePropertyMask* email_property_filters = + search_spec->add_type_property_filters(); + email_property_filters->set_schema_type("unknown"); + email_property_filters->add_paths("sender.name"); + email_property_filters->add_paths("subject"); + + auto result_spec = std::make_unique<ResultSpecProto>(); + + auto scoring_spec = std::make_unique<ScoringSpecProto>(); + *scoring_spec = GetDefaultScoringSpec(); + SearchResultProto results = + icing.Search(*search_spec, *scoring_spec, *result_spec); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(2)); + + // 3. Verify that both the documents are returned since each of them have the + // word 'hello' in at least 1 property. The second document being returned + // confirms that the body field was searched and the specified property + // filters were not applied to the Email schema type. + EXPECT_THAT(results.results(0).document(), + EqualsProto(document_two)); + EXPECT_THAT(results.results(1).document(), + EqualsProto(document_one)); +} + +TEST_P(IcingSearchEngineSearchTest, SearchWithEmptyPropertyFilter) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreateMessageSchema()).status(), + ProtoIsOk()); + + // 1. Add two email documents + DocumentProto document_one = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetCreationTimestampMs(1000) + .SetSchema("Message") + .AddStringProperty("body", "Hello World!") + .Build(); + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + + // 2. Issue a query with empty property filter for Message schema. + auto search_spec = std::make_unique<SearchSpecProto>(); + search_spec->set_term_match_type(TermMatchType::PREFIX); + search_spec->set_query("hello"); + search_spec->set_search_type(GetParam()); + TypePropertyMask* message_property_filters = + search_spec->add_type_property_filters(); + message_property_filters->set_schema_type("Message"); + + auto result_spec = std::make_unique<ResultSpecProto>(); + + auto scoring_spec = std::make_unique<ScoringSpecProto>(); + *scoring_spec = GetDefaultScoringSpec(); + SearchResultProto results = + icing.Search(*search_spec, *scoring_spec, *result_spec); + EXPECT_THAT(results.status(), ProtoIsOk()); + + // 3. Verify that no documents are returned. Although 'hello' is present in + // the indexed document, it shouldn't be returned since the Message property + // filter doesn't allow any properties to be searched. + ASSERT_THAT(results.results(), IsEmpty()); +} + +TEST_P(IcingSearchEngineSearchTest, + SearchWithPropertyFilterHavingInvalidProperty) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreateMessageSchema()).status(), + ProtoIsOk()); + + // 1. Add two email documents + DocumentProto document_one = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetCreationTimestampMs(1000) + .SetSchema("Message") + .AddStringProperty("body", "Hello World!") + .Build(); + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + + // 2. Issue a query with property filter having invalid/unknown property for + // Message schema. + auto search_spec = std::make_unique<SearchSpecProto>(); + search_spec->set_term_match_type(TermMatchType::PREFIX); + search_spec->set_query("hello"); + search_spec->set_search_type(GetParam()); + TypePropertyMask* message_property_filters = + search_spec->add_type_property_filters(); + message_property_filters->set_schema_type("Message"); + message_property_filters->add_paths("unknown"); + + auto result_spec = std::make_unique<ResultSpecProto>(); + + auto scoring_spec = std::make_unique<ScoringSpecProto>(); + *scoring_spec = GetDefaultScoringSpec(); + SearchResultProto results = + icing.Search(*search_spec, *scoring_spec, *result_spec); + EXPECT_THAT(results.status(), ProtoIsOk()); + + // 3. Verify that no documents are returned. Although 'hello' is present in + // the indexed document, it shouldn't be returned since the Message property + // filter doesn't allow any valid properties to be searched. Any + // invalid/unknown properties specified in the property filters will be + // ignored while searching. + ASSERT_THAT(results.results(), IsEmpty()); +} + +TEST_P(IcingSearchEngineSearchTest, SearchWithPropertyFiltersWithNesting) { + IcingSearchEngine icing(GetDefaultIcingOptions(), GetTestJniCache()); + ASSERT_THAT(icing.Initialize().status(), ProtoIsOk()); + ASSERT_THAT(icing.SetSchema(CreatePersonAndEmailSchema()).status(), + ProtoIsOk()); + + // 1. Add two email documents + DocumentProto document_one = + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetCreationTimestampMs(1000) + .SetSchema("Email") + .AddDocumentProperty( + "sender", + DocumentBuilder() + .SetKey("namespace", "uri1") + .SetSchema("Person") + .AddStringProperty("name", "Meg Ryan") + .AddStringProperty("emailAddress", "hellogirl@aol.com") + .Build()) + .AddStringProperty("subject", "Hello World!") + .AddStringProperty( + "body", "Oh what a beautiful morning! Oh what a beautiful day!") + .Build(); + ASSERT_THAT(icing.Put(document_one).status(), ProtoIsOk()); + + DocumentProto document_two = + DocumentBuilder() + .SetKey("namespace", "uri2") + .SetCreationTimestampMs(1000) + .SetSchema("Email") + .AddDocumentProperty( + "sender", DocumentBuilder() + .SetKey("namespace", "uri2") + .SetSchema("Person") + .AddStringProperty("name", "Tom Hanks") + .AddStringProperty("emailAddress", "ny152@aol.com") + .Build()) + .AddStringProperty("subject", "Goodnight Moon!") + .AddStringProperty("body", + "Count all the sheep and tell them 'Hello'.") + .Build(); + ASSERT_THAT(icing.Put(document_two).status(), ProtoIsOk()); + + // 2. Issue a query with property filter of sender.emailAddress for the Email + // schema type. + auto search_spec = std::make_unique<SearchSpecProto>(); + search_spec->set_term_match_type(TermMatchType::PREFIX); + search_spec->set_query("hello"); + search_spec->set_search_type(GetParam()); + TypePropertyMask* email_property_filters = + search_spec->add_type_property_filters(); + email_property_filters->set_schema_type("Email"); + email_property_filters->add_paths("sender.emailAddress"); + + auto result_spec = std::make_unique<ResultSpecProto>(); + + auto scoring_spec = std::make_unique<ScoringSpecProto>(); + *scoring_spec = GetDefaultScoringSpec(); + SearchResultProto results = + icing.Search(*search_spec, *scoring_spec, *result_spec); + EXPECT_THAT(results.status(), ProtoIsOk()); + EXPECT_THAT(results.results(), SizeIs(1)); + + // 3. Verify that only the first document is returned since the second + // document doesn't contain the word 'hello' in sender.emailAddress. The first + // document being returned confirms that the nested property + // sender.emailAddress was actually searched. + EXPECT_THAT(results.results(0).document(), + EqualsProto(document_one)); +} + TEST_P(IcingSearchEngineSearchTest, QueryStatsProtoTest) { auto fake_clock = std::make_unique<FakeClock>(); fake_clock->SetTimerElapsedMilliseconds(5); diff --git a/icing/index/index-processor_benchmark.cc b/icing/index/index-processor_benchmark.cc index b6d3c29..8766f0b 100644 --- a/icing/index/index-processor_benchmark.cc +++ b/icing/index/index-processor_benchmark.cc @@ -150,7 +150,9 @@ DocumentProto CreateDocumentWithHiragana(int content_length) { std::unique_ptr<Index> CreateIndex(const IcingFilesystem& icing_filesystem, const Filesystem& filesystem, const std::string& index_dir) { - Index::Options options(index_dir, /*index_merge_size=*/1024 * 1024 * 10); + Index::Options options(index_dir, /*index_merge_size=*/1024 * 1024 * 10, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); return Index::Create(options, &filesystem, &icing_filesystem).ValueOrDie(); } @@ -227,6 +229,7 @@ void BM_IndexDocumentWithOneProperty(benchmark::State& state) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<NumericIndex<int64_t>> integer_index, IntegerIndex::Create(filesystem, integer_index_dir, + IntegerIndex::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv=*/true)); language_segmenter_factory::SegmenterOptions options(ULOC_US); std::unique_ptr<LanguageSegmenter> language_segmenter = @@ -302,6 +305,7 @@ void BM_IndexDocumentWithTenProperties(benchmark::State& state) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<NumericIndex<int64_t>> integer_index, IntegerIndex::Create(filesystem, integer_index_dir, + IntegerIndex::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv=*/true)); language_segmenter_factory::SegmenterOptions options(ULOC_US); std::unique_ptr<LanguageSegmenter> language_segmenter = @@ -378,6 +382,7 @@ void BM_IndexDocumentWithDiacriticLetters(benchmark::State& state) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<NumericIndex<int64_t>> integer_index, IntegerIndex::Create(filesystem, integer_index_dir, + IntegerIndex::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv=*/true)); language_segmenter_factory::SegmenterOptions options(ULOC_US); std::unique_ptr<LanguageSegmenter> language_segmenter = @@ -454,6 +459,7 @@ void BM_IndexDocumentWithHiragana(benchmark::State& state) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<NumericIndex<int64_t>> integer_index, IntegerIndex::Create(filesystem, integer_index_dir, + IntegerIndex::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv=*/true)); language_segmenter_factory::SegmenterOptions options(ULOC_US); std::unique_ptr<LanguageSegmenter> language_segmenter = diff --git a/icing/index/index-processor_test.cc b/icing/index/index-processor_test.cc index 0a0108d..ba4ece3 100644 --- a/icing/index/index-processor_test.cc +++ b/icing/index/index-processor_test.cc @@ -40,8 +40,8 @@ #include "icing/index/numeric/numeric-index.h" #include "icing/index/string-section-indexing-handler.h" #include "icing/index/term-property-id.h" +#include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id-join-indexing-handler.h" -#include "icing/join/qualified-id-type-joinable-index.h" #include "icing/legacy/index/icing-filesystem.h" #include "icing/legacy/index/icing-mock-filesystem.h" #include "icing/portable/platform.h" @@ -167,19 +167,24 @@ class IndexProcessorTest : public Test { schema_store_dir_ = base_dir_ + "/schema_store"; doc_store_dir_ = base_dir_ + "/doc_store"; - Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024); + Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, &icing_filesystem_)); ICING_ASSERT_OK_AND_ASSIGN( - integer_index_, IntegerIndex::Create(filesystem_, integer_index_dir_, - /*pre_mapping_fbv=*/false)); + integer_index_, + IntegerIndex::Create( + filesystem_, integer_index_dir_, + IntegerIndex::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv=*/false)); ICING_ASSERT_OK_AND_ASSIGN( qualified_id_join_index_, - QualifiedIdTypeJoinableIndex::Create( - filesystem_, qualified_id_join_index_dir_, - /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/false)); + QualifiedIdJoinIndex::Create(filesystem_, qualified_id_join_index_dir_, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false)); language_segmenter_factory::SegmenterOptions segmenter_options(ULOC_US); ICING_ASSERT_OK_AND_ASSIGN( @@ -277,13 +282,14 @@ class IndexProcessorTest : public Test { ASSERT_TRUE(filesystem_.CreateDirectoryRecursively(doc_store_dir_.c_str())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, doc_store_dir_, &fake_clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); doc_store_ = std::move(create_result.document_store); ICING_ASSERT_OK_AND_ASSIGN( @@ -297,14 +303,13 @@ class IndexProcessorTest : public Test { &fake_clock_, integer_index_.get())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<QualifiedIdJoinIndexingHandler> - qualified_id_joinable_property_indexing_handler, + qualified_id_join_indexing_handler, QualifiedIdJoinIndexingHandler::Create(&fake_clock_, qualified_id_join_index_.get())); std::vector<std::unique_ptr<DataIndexingHandler>> handlers; handlers.push_back(std::move(string_section_indexing_handler)); handlers.push_back(std::move(integer_section_indexing_handler)); - handlers.push_back( - std::move(qualified_id_joinable_property_indexing_handler)); + handlers.push_back(std::move(qualified_id_join_indexing_handler)); index_processor_ = std::make_unique<IndexProcessor>(std::move(handlers), &fake_clock_); @@ -339,7 +344,7 @@ class IndexProcessorTest : public Test { std::unique_ptr<Index> index_; std::unique_ptr<NumericIndex<int64_t>> integer_index_; - std::unique_ptr<QualifiedIdTypeJoinableIndex> qualified_id_join_index_; + std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index_; std::unique_ptr<LanguageSegmenter> lang_segmenter_; std::unique_ptr<Normalizer> normalizer_; std::unique_ptr<SchemaStore> schema_store_; @@ -827,16 +832,14 @@ TEST_F(IndexProcessorTest, OutOfOrderDocumentIdsInRecoveryMode) { integer_section_indexing_handler, IntegerSectionIndexingHandler::Create( &fake_clock_, integer_index_.get())); - ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdJoinIndexingHandler> - qualified_id_joinable_property_indexing_handler, - QualifiedIdJoinIndexingHandler::Create(&fake_clock_, - qualified_id_join_index_.get())); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<QualifiedIdJoinIndexingHandler> + qualified_id_join_indexing_handler, + QualifiedIdJoinIndexingHandler::Create( + &fake_clock_, qualified_id_join_index_.get())); std::vector<std::unique_ptr<DataIndexingHandler>> handlers; handlers.push_back(std::move(string_section_indexing_handler)); handlers.push_back(std::move(integer_section_indexing_handler)); - handlers.push_back( - std::move(qualified_id_joinable_property_indexing_handler)); + handlers.push_back(std::move(qualified_id_join_indexing_handler)); IndexProcessor index_processor(std::move(handlers), &fake_clock_, /*recovery_mode=*/true); @@ -969,7 +972,9 @@ TEST_F(IndexProcessorTest, IndexingDocAutomaticMerge) { TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), document)); Index::Options options(index_dir_, - /*index_merge_size=*/document.ByteSizeLong() * 100); + /*index_merge_size=*/document.ByteSizeLong() * 100, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/64); ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, &icing_filesystem_)); @@ -1032,7 +1037,9 @@ TEST_F(IndexProcessorTest, IndexingDocMergeFailureResets) { // 2. Recreate the index with the mock filesystem and a merge size that will // only allow one document to be added before requiring a merge. Index::Options options(index_dir_, - /*index_merge_size=*/document.ByteSizeLong()); + /*index_merge_size=*/document.ByteSizeLong(), + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/16); ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, mock_icing_filesystem_.get())); diff --git a/icing/index/index.cc b/icing/index/index.cc index 19edbb6..31dcc7e 100644 --- a/icing/index/index.cc +++ b/icing/index/index.cc @@ -14,31 +14,38 @@ #include "icing/index/index.h" +#include <algorithm> +#include <cstddef> #include <cstdint> #include <memory> #include <string> #include <utility> +#include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" +#include "icing/file/filesystem.h" #include "icing/index/hit/hit.h" #include "icing/index/iterator/doc-hit-info-iterator-or.h" #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/index/lite/doc-hit-info-iterator-term-lite.h" #include "icing/index/lite/lite-index.h" #include "icing/index/main/doc-hit-info-iterator-term-main.h" +#include "icing/index/main/main-index.h" #include "icing/index/term-id-codec.h" -#include "icing/index/term-property-id.h" +#include "icing/index/term-metadata.h" #include "icing/legacy/core/icing-string-util.h" #include "icing/legacy/index/icing-dynamic-trie.h" #include "icing/legacy/index/icing-filesystem.h" +#include "icing/proto/scoring.pb.h" #include "icing/proto/storage.pb.h" #include "icing/proto/term.pb.h" #include "icing/schema/section.h" #include "icing/scoring/ranker.h" #include "icing/store/document-id.h" +#include "icing/store/suggestion-result-checker.h" #include "icing/util/logging.h" #include "icing/util/status-macros.h" @@ -59,7 +66,9 @@ libtextclassifier3::StatusOr<LiteIndex::Options> CreateLiteIndexOptions( options.index_merge_size)); } return LiteIndex::Options(options.base_dir + "/idx/lite.", - options.index_merge_size); + options.index_merge_size, + options.lite_index_sort_at_indexing, + options.lite_index_sort_size); } std::string MakeMainIndexFilepath(const std::string& base_dir) { @@ -151,9 +160,17 @@ libtextclassifier3::StatusOr<std::unique_ptr<Index>> Index::Create( IcingDynamicTrie::max_value_index(GetMainLexiconOptions()), IcingDynamicTrie::max_value_index( lite_index_options.lexicon_options))); + ICING_ASSIGN_OR_RETURN( std::unique_ptr<LiteIndex> lite_index, LiteIndex::Create(lite_index_options, icing_filesystem)); + // Sort the lite index if we've enabled sorting the HitBuffer at indexing + // time, and there's an unsorted tail exceeding the threshold. + if (options.lite_index_sort_at_indexing && + lite_index->HasUnsortedHitsExceedingSortThreshold()) { + lite_index->SortHits(); + } + ICING_ASSIGN_OR_RETURN( std::unique_ptr<MainIndex> main_index, MainIndex::Create(MakeMainIndexFilepath(options.base_dir), filesystem, diff --git a/icing/index/index.h b/icing/index/index.h index c170278..32ea97b 100644 --- a/icing/index/index.h +++ b/icing/index/index.h @@ -18,8 +18,9 @@ #include <cstdint> #include <memory> #include <string> -#include <unordered_set> +#include <unordered_map> #include <utility> +#include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/text_classifier/lib3/utils/base/statusor.h" @@ -27,6 +28,7 @@ #include "icing/index/hit/hit.h" #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/index/lite/lite-index.h" +#include "icing/index/lite/term-id-hit-pair.h" #include "icing/index/main/main-index-merger.h" #include "icing/index/main/main-index.h" #include "icing/index/term-id-codec.h" @@ -40,7 +42,7 @@ #include "icing/store/document-id.h" #include "icing/store/namespace-id.h" #include "icing/store/suggestion-result-checker.h" -#include "icing/util/crc32.h" +#include "icing/util/status-macros.h" namespace icing { namespace lib { @@ -68,11 +70,18 @@ namespace lib { class Index { public: struct Options { - explicit Options(const std::string& base_dir, uint32_t index_merge_size) - : base_dir(base_dir), index_merge_size(index_merge_size) {} + explicit Options(const std::string& base_dir, uint32_t index_merge_size, + bool lite_index_sort_at_indexing, + uint32_t lite_index_sort_size) + : base_dir(base_dir), + index_merge_size(index_merge_size), + lite_index_sort_at_indexing(lite_index_sort_at_indexing), + lite_index_sort_size(lite_index_sort_size) {} std::string base_dir; int32_t index_merge_size; + bool lite_index_sort_at_indexing; + int32_t lite_index_sort_size; }; // Creates an instance of Index in the directory pointed by file_dir. @@ -279,6 +288,19 @@ class Index { return lite_index_->Reset(); } + // Whether the LiteIndex HitBuffer requires sorting. This is only true if + // Icing has enabled sorting during indexing time, and the HitBuffer's + // unsorted tail has exceeded the lite_index_sort_size. + bool LiteIndexNeedSort() const { + return options_.lite_index_sort_at_indexing && + lite_index_->HasUnsortedHitsExceedingSortThreshold(); + } + + // Sorts the LiteIndex HitBuffer. + void SortLiteIndex() { + lite_index_->SortHits(); + } + // Reduces internal file sizes by reclaiming space of deleted documents. // new_last_added_document_id will be used to update the last added document // id in the lite index. diff --git a/icing/index/index_test.cc b/icing/index/index_test.cc index d563bcb..b823535 100644 --- a/icing/index/index_test.cc +++ b/icing/index/index_test.cc @@ -58,6 +58,7 @@ using ::testing::Eq; using ::testing::Ge; using ::testing::Gt; using ::testing::IsEmpty; +using ::testing::IsFalse; using ::testing::IsTrue; using ::testing::Ne; using ::testing::NiceMock; @@ -75,7 +76,9 @@ class IndexTest : public Test { protected: void SetUp() override { index_dir_ = GetTestTempDir() + "/index_test/"; - Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024); + Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, &icing_filesystem_)); } @@ -146,7 +149,9 @@ MATCHER_P2(EqualsTermMetadata, content, hit_count, "") { } TEST_F(IndexTest, CreationWithNullPointerShouldFail) { - Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024); + Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); EXPECT_THAT( Index::Create(options, &filesystem_, /*icing_filesystem=*/nullptr), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); @@ -192,6 +197,36 @@ TEST_F(IndexTest, EmptyIndexAfterMerge) { StatusIs(libtextclassifier3::StatusCode::RESOURCE_EXHAUSTED)); } +TEST_F(IndexTest, CreationWithLiteIndexSortAtIndexingEnabledShouldSort) { + // Make the index with lite_index_sort_at_indexing=false and a very small sort + // threshold. + Index::Options options(index_dir_, /*index_merge_size=*/1024, + /*lite_index_sort_at_indexing=*/false, + /*lite_index_sort_size=*/16); + ICING_ASSERT_OK_AND_ASSIGN( + index_, Index::Create(options, &filesystem_, &icing_filesystem_)); + + Index::Editor edit = index_->Edit( + kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); + ASSERT_THAT(edit.BufferTerm("foo"), IsOk()); + ASSERT_THAT(edit.BufferTerm("bar"), IsOk()); + ASSERT_THAT(edit.BufferTerm("baz"), IsOk()); + ASSERT_THAT(edit.IndexAllBufferedTerms(), IsOk()); + + // Persist and recreate the index with lite_index_sort_at_indexing=true + ASSERT_THAT(index_->PersistToDisk(), IsOk()); + options = Index::Options(index_dir_, /*index_merge_size=*/1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/16); + ICING_ASSERT_OK_AND_ASSIGN( + index_, Index::Create(options, &filesystem_, &icing_filesystem_)); + + // Check that the index is sorted after recreating with + // lite_index_sort_at_indexing, with the unsorted HitBuffer exceeding the sort + // threshold. + EXPECT_THAT(index_->LiteIndexNeedSort(), IsFalse()); +} + TEST_F(IndexTest, AdvancePastEnd) { Index::Editor edit = index_->Edit( kDocumentId0, kSectionId2, TermMatchType::EXACT_ONLY, /*namespace_id=*/0); @@ -967,7 +1002,9 @@ TEST_F(IndexTest, NonAsciiTermsAfterMerge) { TEST_F(IndexTest, FullIndex) { // Make a smaller index so that it's easier to fill up. - Index::Options options(index_dir_, /*index_merge_size=*/1024); + Index::Options options(index_dir_, /*index_merge_size=*/1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/64); ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, &icing_filesystem_)); @@ -1035,7 +1072,9 @@ TEST_F(IndexTest, FullIndex) { TEST_F(IndexTest, FullIndexMerge) { // Make a smaller index so that it's easier to fill up. - Index::Options options(index_dir_, /*index_merge_size=*/1024); + Index::Options options(index_dir_, /*index_merge_size=*/1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/64); ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, &icing_filesystem_)); @@ -1368,7 +1407,9 @@ TEST_F(IndexTest, IndexCreateIOFailure) { NiceMock<IcingMockFilesystem> mock_icing_filesystem; ON_CALL(mock_icing_filesystem, CreateDirectoryRecursively) .WillByDefault(Return(false)); - Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024); + Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); EXPECT_THAT(Index::Create(options, &filesystem_, &mock_icing_filesystem), StatusIs(libtextclassifier3::StatusCode::INTERNAL)); } @@ -1399,7 +1440,9 @@ TEST_F(IndexTest, IndexCreateCorruptionFailure) { IsTrue()); // Recreate the index. - Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024); + Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); EXPECT_THAT(Index::Create(options, &filesystem_, &icing_filesystem_), StatusIs(libtextclassifier3::StatusCode::DATA_LOSS)); } @@ -1417,7 +1460,9 @@ TEST_F(IndexTest, IndexPersistence) { index_.reset(); // Recreate the index. - Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024); + Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, &icing_filesystem_)); @@ -1446,7 +1491,9 @@ TEST_F(IndexTest, IndexPersistenceAfterMerge) { index_.reset(); // Recreate the index. - Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024); + Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, &icing_filesystem_)); @@ -1463,7 +1510,8 @@ TEST_F(IndexTest, IndexPersistenceAfterMerge) { TEST_F(IndexTest, InvalidHitBufferSize) { Index::Options options( - index_dir_, /*index_merge_size=*/std::numeric_limits<uint32_t>::max()); + index_dir_, /*index_merge_size=*/std::numeric_limits<uint32_t>::max(), + /*lite_index_sort_at_indexing=*/true, /*lite_index_sort_size=*/1024 * 8); EXPECT_THAT(Index::Create(options, &filesystem_, &icing_filesystem_), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } diff --git a/icing/index/integer-section-indexing-handler_test.cc b/icing/index/integer-section-indexing-handler_test.cc index 96e21ca..91cc06f 100644 --- a/icing/index/integer-section-indexing-handler_test.cc +++ b/icing/index/integer-section-indexing-handler_test.cc @@ -106,6 +106,7 @@ class IntegerSectionIndexingHandlerTest : public ::testing::Test { ICING_ASSERT_OK_AND_ASSIGN( integer_index_, IntegerIndex::Create(filesystem_, integer_index_working_path_, + /*num_data_threshold_for_bucket_split=*/65536, /*pre_mapping_fbv=*/false)); language_segmenter_factory::SegmenterOptions segmenter_options(ULOC_US); @@ -169,6 +170,8 @@ class IntegerSectionIndexingHandlerTest : public ::testing::Test { schema_store_.get(), /*force_recovery_and_revalidate_documents=*/false, /*namespace_id_fingerprint=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, PortableFileBackedProtoLog< DocumentWrapper>::kDeflateCompressionLevel, /*initialize_stats=*/nullptr)); diff --git a/icing/index/iterator/doc-hit-info-iterator-filter_test.cc b/icing/index/iterator/doc-hit-info-iterator-filter_test.cc index d8839dc..d93fd02 100644 --- a/icing/index/iterator/doc-hit-info-iterator-filter_test.cc +++ b/icing/index/iterator/doc-hit-info-iterator-filter_test.cc @@ -55,7 +55,8 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( return DocumentStore::Create( filesystem, base_dir, clock, schema_store, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, /*initialize_stats=*/nullptr); } diff --git a/icing/index/iterator/doc-hit-info-iterator-property-in-schema_test.cc b/icing/index/iterator/doc-hit-info-iterator-property-in-schema_test.cc index df5ddf5..47f5cc5 100644 --- a/icing/index/iterator/doc-hit-info-iterator-property-in-schema_test.cc +++ b/icing/index/iterator/doc-hit-info-iterator-property-in-schema_test.cc @@ -97,13 +97,14 @@ class DocHitInfoIteratorPropertyInSchemaTest : public ::testing::Test { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, test_dir_, &fake_clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); } diff --git a/icing/index/iterator/doc-hit-info-iterator-section-restrict.cc b/icing/index/iterator/doc-hit-info-iterator-section-restrict.cc index 227a185..b850a9b 100644 --- a/icing/index/iterator/doc-hit-info-iterator-section-restrict.cc +++ b/icing/index/iterator/doc-hit-info-iterator-section-restrict.cc @@ -42,8 +42,99 @@ DocHitInfoIteratorSectionRestrict::DocHitInfoIteratorSectionRestrict( : delegate_(std::move(delegate)), document_store_(*document_store), schema_store_(*schema_store), - target_sections_(std::move(target_sections)), - current_time_ms_(current_time_ms) {} + current_time_ms_(current_time_ms) { + type_property_filters_[std::string(SchemaStore::kSchemaTypeWildcard)] = + std::move(target_sections); +} + +DocHitInfoIteratorSectionRestrict::DocHitInfoIteratorSectionRestrict( + std::unique_ptr<DocHitInfoIterator> delegate, + const DocumentStore* document_store, const SchemaStore* schema_store, + const SearchSpecProto& search_spec, + int64_t current_time_ms) + : delegate_(std::move(delegate)), + document_store_(*document_store), + schema_store_(*schema_store), + current_time_ms_(current_time_ms) { + // TODO(b/294274922): Add support for polymorphism in type property filters. + for (const TypePropertyMask& type_property_mask : + search_spec.type_property_filters()) { + type_property_filters_[type_property_mask.schema_type()] = + std::set<std::string>(type_property_mask.paths().begin(), + type_property_mask.paths().end()); + } +} + +DocHitInfoIteratorSectionRestrict::DocHitInfoIteratorSectionRestrict( + std::unique_ptr<DocHitInfoIterator> delegate, + const DocumentStore* document_store, const SchemaStore* schema_store, + std::unordered_map<std::string, std::set<std::string>> + type_property_filters, + std::unordered_map<std::string, SectionIdMask> type_property_masks, + int64_t current_time_ms) + : delegate_(std::move(delegate)), + document_store_(*document_store), + schema_store_(*schema_store), + current_time_ms_(current_time_ms), + type_property_filters_(std::move(type_property_filters)), + type_property_masks_(std::move(type_property_masks)) {} + +SectionIdMask DocHitInfoIteratorSectionRestrict::GenerateSectionMask( + const std::string& schema_type, + const std::set<std::string>& target_sections) const { + SectionIdMask section_mask = kSectionIdMaskNone; + auto section_metadata_list_or = + schema_store_.GetSectionMetadata(schema_type); + if (!section_metadata_list_or.ok()) { + // The current schema doesn't have section metadata. + return kSectionIdMaskNone; + } + const std::vector<SectionMetadata>* section_metadata_list = + section_metadata_list_or.ValueOrDie(); + for (const SectionMetadata& section_metadata : *section_metadata_list) { + if (target_sections.find(section_metadata.path) != + target_sections.end()) { + section_mask |= UINT64_C(1) << section_metadata.id; + } + } + return section_mask; +} + +SectionIdMask DocHitInfoIteratorSectionRestrict:: + ComputeAndCacheSchemaTypeAllowedSectionsMask( + const std::string& schema_type) { + if (const auto type_property_mask_itr = + type_property_masks_.find(schema_type); + type_property_mask_itr != type_property_masks_.end()) { + return type_property_mask_itr->second; + } + + // Section id mask of schema_type is never calculated before, so + // calculate it here and put it into type_property_masks_. + // - If type property filters of schema_type or wildcard (*) are + // specified, then create a mask according to the filters. + // - Otherwise, create a mask to match all properties. + SectionIdMask new_section_id_mask = kSectionIdMaskAll; + if (const auto itr = type_property_filters_.find(schema_type); + itr != type_property_filters_.end()) { + // Property filters defined for given schema type + new_section_id_mask = GenerateSectionMask( + schema_type, itr->second); + } else if (const auto wildcard_itr = type_property_filters_.find( + std::string(SchemaStore::kSchemaTypeWildcard)); + wildcard_itr != type_property_filters_.end()) { + // Property filters defined for wildcard entry + new_section_id_mask = GenerateSectionMask( + schema_type, wildcard_itr->second); + } else { + // Do not cache the section mask if no property filters apply to this schema + // type to avoid taking up unnecessary space. + return kSectionIdMaskAll; + } + + type_property_masks_[schema_type] = new_section_id_mask; + return new_section_id_mask; +} libtextclassifier3::Status DocHitInfoIteratorSectionRestrict::Advance() { doc_hit_info_ = DocHitInfo(kInvalidDocumentId); @@ -63,32 +154,32 @@ libtextclassifier3::Status DocHitInfoIteratorSectionRestrict::Advance() { // Guaranteed that the DocumentFilterData exists at this point SchemaTypeId schema_type_id = data_optional.value().schema_type_id(); - - // A hit can be in multiple sections at once, need to check which of the - // section ids match the target sections - while (section_id_mask != 0) { - // There was a hit in this section id - SectionId section_id = __builtin_ctzll(section_id_mask); - - auto section_metadata_or = - schema_store_.GetSectionMetadata(schema_type_id, section_id); - - if (section_metadata_or.ok()) { - const SectionMetadata* section_metadata = - section_metadata_or.ValueOrDie(); - - if (target_sections_.find(section_metadata->path) != - target_sections_.end()) { - // The hit was in the target section name, return OK/found - hit_intersect_section_ids_mask_ |= UINT64_C(1) << section_id; - } - } - - // Mark this section as checked - section_id_mask &= ~(UINT64_C(1) << section_id); + auto schema_type_or = schema_store_.GetSchemaType(schema_type_id); + if (!schema_type_or.ok()) { + // Ran into error retrieving schema type, skip + continue; } + const std::string* schema_type = std::move(schema_type_or).ValueOrDie(); + SectionIdMask allowed_sections_mask = + ComputeAndCacheSchemaTypeAllowedSectionsMask(*schema_type); - if (hit_intersect_section_ids_mask_ != kSectionIdMaskNone) { + // A hit can be in multiple sections at once, need to check which of the + // section ids match the sections allowed by type_property_masks_. This can + // be done by doing a bitwise and of the section_id_mask in the doc hit and + // the allowed_sections_mask. + hit_intersect_section_ids_mask_ = section_id_mask & allowed_sections_mask; + + // Return this document if: + // - the sectionIdMask is not empty after applying property filters, or + // - no property filters apply for its schema type (allowed_sections_mask + // == kSectionIdMaskAll). This is needed to ensure that in case of empty + // query (which uses doc-hit-info-iterator-all-document-id), where + // section_id_mask (and hence hit_intersect_section_ids_mask_) is + // kSectionIdMaskNone, doc hits with no property restrictions don't get + // filtered out. Doc hits for schema types for whom property filters are + // specified will still get filtered out. + if (allowed_sections_mask == kSectionIdMaskAll + || hit_intersect_section_ids_mask_ != kSectionIdMaskNone) { doc_hit_info_ = delegate_->doc_hit_info(); doc_hit_info_.set_hit_section_ids_mask(hit_intersect_section_ids_mask_); return libtextclassifier3::Status::OK; @@ -104,16 +195,36 @@ libtextclassifier3::StatusOr<DocHitInfoIterator::TrimmedNode> DocHitInfoIteratorSectionRestrict::TrimRightMostNode() && { ICING_ASSIGN_OR_RETURN(TrimmedNode trimmed_delegate, std::move(*delegate_).TrimRightMostNode()); + // TrimRightMostNode is only used by suggestion processor to process query + // expression, so an entry for wildcard should always be present in + // type_property_filters_ when code flow reaches here. If the InternalError + // below is returned, that means TrimRightMostNode hasn't been called in the + // right context. + const auto it = type_property_filters_.find("*"); + if (it == type_property_filters_.end()) { + return absl_ports::InternalError( + "A wildcard entry should always be present in type property filters " + "whenever TrimRightMostNode() is called for " + "DocHitInfoIteratorSectionRestrict"); + } + std::set<std::string>& target_sections = it->second; + if (target_sections.empty()) { + return absl_ports::InternalError( + "Target sections should not be empty whenever TrimRightMostNode() is " + "called for DocHitInfoIteratorSectionRestrict"); + } if (trimmed_delegate.iterator_ == nullptr) { // TODO(b/228240987): Update TrimmedNode and downstream code to handle // multiple section restricts. - trimmed_delegate.target_section_ = std::move(*target_sections_.begin()); + trimmed_delegate.target_section_ = std::move(*target_sections.begin()); return trimmed_delegate; } trimmed_delegate.iterator_ = - std::make_unique<DocHitInfoIteratorSectionRestrict>( + std::unique_ptr<DocHitInfoIteratorSectionRestrict>( + new DocHitInfoIteratorSectionRestrict( std::move(trimmed_delegate.iterator_), &document_store_, - &schema_store_, std::move(target_sections_), current_time_ms_); + &schema_store_, std::move(type_property_filters_), + std::move(type_property_masks_), current_time_ms_)); return std::move(trimmed_delegate); } @@ -126,8 +237,14 @@ int32_t DocHitInfoIteratorSectionRestrict::GetNumLeafAdvanceCalls() const { } std::string DocHitInfoIteratorSectionRestrict::ToString() const { - return absl_ports::StrCat("(", absl_ports::StrJoin(target_sections_, ","), - "): ", delegate_->ToString()); + std::string output = ""; + for (auto it = type_property_filters_.cbegin(); + it != type_property_filters_.cend(); it++) { + std::string paths = absl_ports::StrJoin(it->second, ","); + output += (it->first) + ":" + (paths) + "; "; + } + std::string result = "{" + output.substr(0, output.size() - 2) + "}: "; + return absl_ports::StrCat(result, delegate_->ToString()); } } // namespace lib diff --git a/icing/index/iterator/doc-hit-info-iterator-section-restrict.h b/icing/index/iterator/doc-hit-info-iterator-section-restrict.h index 58dd120..5d44ed7 100644 --- a/icing/index/iterator/doc-hit-info-iterator-section-restrict.h +++ b/icing/index/iterator/doc-hit-info-iterator-section-restrict.h @@ -19,10 +19,13 @@ #include <memory> #include <string> #include <string_view> +#include <unordered_map> #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/schema/schema-store.h" +#include "icing/schema/section.h" +#include "icing/store/document-filter-data.h" #include "icing/store/document-store.h" namespace icing { @@ -44,6 +47,12 @@ class DocHitInfoIteratorSectionRestrict : public DocHitInfoIterator { const DocumentStore* document_store, const SchemaStore* schema_store, std::set<std::string> target_sections, int64_t current_time_ms); + explicit DocHitInfoIteratorSectionRestrict( + std::unique_ptr<DocHitInfoIterator> delegate, + const DocumentStore* document_store, const SchemaStore* schema_store, + const SearchSpecProto& search_spec, + int64_t current_time_ms); + libtextclassifier3::Status Advance() override; libtextclassifier3::StatusOr<TrimmedNode> TrimRightMostNode() && override; @@ -72,12 +81,51 @@ class DocHitInfoIteratorSectionRestrict : public DocHitInfoIterator { } private: + explicit DocHitInfoIteratorSectionRestrict( + std::unique_ptr<DocHitInfoIterator> delegate, + const DocumentStore* document_store, const SchemaStore* schema_store, + std::unordered_map<std::string, std::set<std::string>> + type_property_filters, + std::unordered_map<std::string, SectionIdMask> type_property_masks, + int64_t current_time_ms); + // Calculates the section mask of allowed sections(determined by the property + // filters map) for the given schema type and caches the same for any future + // calls. + // + // Returns: + // - If type_property_filters_ has an entry for the given schema type or + // wildcard(*), return a bitwise or of section IDs in the schema type that + // that are also present in the relevant filter list. + // - Otherwise, return kSectionIdMaskAll. + SectionIdMask ComputeAndCacheSchemaTypeAllowedSectionsMask( + const std::string& schema_type); + // Generates a section mask for the given schema type and the target sections. + // + // Returns: + // - A bitwise or of section IDs in the schema_type that that are also + // present in the target_sections list. + // - If none of the sections in the schema_type are present in the + // target_sections list, return kSectionIdMaskNone. + // This is done by doing a bitwise or of the target section ids for the given + // schema type. + SectionIdMask GenerateSectionMask(const std::string& schema_type, + const std::set<std::string>& + target_sections) const; + std::unique_ptr<DocHitInfoIterator> delegate_; const DocumentStore& document_store_; const SchemaStore& schema_store_; - - std::set<std::string> target_sections_; int64_t current_time_ms_; + + // Map of property filters per schema type. Supports wildcard(*) for schema + // type that will apply to all schema types that are not specifically + // specified in the mapping otherwise. + std::unordered_map<std::string, std::set<std::string>> + type_property_filters_; + // Mapping of schema type to the section mask of allowed sections for that + // schema type. This section mask is lazily calculated based on the specified + // property filters and cached for any future use. + std::unordered_map<std::string, SectionIdMask> type_property_masks_; }; } // namespace lib diff --git a/icing/index/iterator/doc-hit-info-iterator-section-restrict_test.cc b/icing/index/iterator/doc-hit-info-iterator-section-restrict_test.cc index c765e6d..1500571 100644 --- a/icing/index/iterator/doc-hit-info-iterator-section-restrict_test.cc +++ b/icing/index/iterator/doc-hit-info-iterator-section-restrict_test.cc @@ -101,13 +101,14 @@ class DocHitInfoIteratorSectionRestrictTest : public ::testing::Test { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, test_dir_, &fake_clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); } @@ -185,7 +186,7 @@ TEST_F(DocHitInfoIteratorSectionRestrictTest, EmptyOriginalIterator) { DocHitInfoIteratorSectionRestrict filtered_iterator( std::move(original_iterator_empty), document_store_.get(), - schema_store_.get(), /*target_sections=*/{}, + schema_store_.get(), /*target_sections=*/std::set<std::string>(), fake_clock_.GetSystemTimeMilliseconds()); EXPECT_THAT(GetDocumentIds(&filtered_iterator), IsEmpty()); @@ -390,7 +391,7 @@ TEST_F(DocHitInfoIteratorSectionRestrictTest, // Create a hit that exists in a different section, so it shouldn't match any // section filters std::vector<DocHitInfo> doc_hit_infos = { - DocHitInfo(document_id, kSectionIdMaskNone << not_matching_section_id)}; + DocHitInfo(document_id, UINT64_C(1) << not_matching_section_id)}; std::unique_ptr<DocHitInfoIterator> original_iterator = std::make_unique<DocHitInfoIteratorDummy>(doc_hit_infos); diff --git a/icing/index/lite/lite-index-options.cc b/icing/index/lite/lite-index-options.cc index 29075f8..8780d45 100644 --- a/icing/index/lite/lite-index-options.cc +++ b/icing/index/lite/lite-index-options.cc @@ -14,6 +14,8 @@ #include "icing/index/lite/lite-index-options.h" +#include <cstdint> + #include "icing/index/lite/term-id-hit-pair.h" namespace icing { @@ -64,9 +66,13 @@ IcingDynamicTrie::Options CalculateTrieOptions(uint32_t hit_buffer_size) { } // namespace LiteIndexOptions::LiteIndexOptions(const std::string& filename_base, - uint32_t hit_buffer_want_merge_bytes) + uint32_t hit_buffer_want_merge_bytes, + bool hit_buffer_sort_at_indexing, + uint32_t hit_buffer_sort_threshold_bytes) : filename_base(filename_base), - hit_buffer_want_merge_bytes(hit_buffer_want_merge_bytes) { + hit_buffer_want_merge_bytes(hit_buffer_want_merge_bytes), + hit_buffer_sort_at_indexing(hit_buffer_sort_at_indexing), + hit_buffer_sort_threshold_bytes(hit_buffer_sort_threshold_bytes) { hit_buffer_size = CalculateHitBufferSize(hit_buffer_want_merge_bytes); lexicon_options = CalculateTrieOptions(hit_buffer_size); display_mappings_options = CalculateTrieOptions(hit_buffer_size); diff --git a/icing/index/lite/lite-index-options.h b/icing/index/lite/lite-index-options.h index ae58802..9f8452c 100644 --- a/icing/index/lite/lite-index-options.h +++ b/icing/index/lite/lite-index-options.h @@ -27,7 +27,9 @@ struct LiteIndexOptions { // hit_buffer_want_merge_bytes and the logic in CalculateHitBufferSize and // CalculateTrieOptions. LiteIndexOptions(const std::string& filename_base, - uint32_t hit_buffer_want_merge_bytes); + uint32_t hit_buffer_want_merge_bytes, + bool hit_buffer_sort_at_indexing, + uint32_t hit_buffer_sort_threshold_bytes); IcingDynamicTrie::Options lexicon_options; IcingDynamicTrie::Options display_mappings_options; @@ -35,6 +37,8 @@ struct LiteIndexOptions { std::string filename_base; uint32_t hit_buffer_want_merge_bytes = 0; uint32_t hit_buffer_size = 0; + bool hit_buffer_sort_at_indexing = false; + uint32_t hit_buffer_sort_threshold_bytes = 0; }; } // namespace lib diff --git a/icing/index/lite/lite-index.cc b/icing/index/lite/lite-index.cc index bf54dec..ec7141a 100644 --- a/icing/index/lite/lite-index.cc +++ b/icing/index/lite/lite-index.cc @@ -36,6 +36,8 @@ #include "icing/index/hit/doc-hit-info.h" #include "icing/index/hit/hit.h" #include "icing/index/lite/lite-index-header.h" +#include "icing/index/lite/term-id-hit-pair.h" +#include "icing/index/term-id-codec.h" #include "icing/index/term-property-id.h" #include "icing/legacy/core/icing-string-util.h" #include "icing/legacy/core/icing-timer.h" @@ -44,10 +46,13 @@ #include "icing/legacy/index/icing-filesystem.h" #include "icing/legacy/index/icing-mmapper.h" #include "icing/proto/debug.pb.h" +#include "icing/proto/scoring.pb.h" #include "icing/proto/storage.pb.h" #include "icing/proto/term.pb.h" #include "icing/schema/section.h" #include "icing/store/document-id.h" +#include "icing/store/namespace-id.h" +#include "icing/store/suggestion-result-checker.h" #include "icing/util/crc32.h" #include "icing/util/logging.h" #include "icing/util/status-macros.h" @@ -160,7 +165,7 @@ libtextclassifier3::Status LiteIndex::Initialize() { } // Set up header. - header_mmap_.Remap(hit_buffer_fd_.get(), 0, header_size()); + header_mmap_.Remap(hit_buffer_fd_.get(), kHeaderFileOffset, header_size()); header_ = std::make_unique<LiteIndex_HeaderImpl>( reinterpret_cast<LiteIndex_HeaderImpl::HeaderData*>( header_mmap_.address())); @@ -175,7 +180,7 @@ libtextclassifier3::Status LiteIndex::Initialize() { UpdateChecksum(); } else { - header_mmap_.Remap(hit_buffer_fd_.get(), 0, header_size()); + header_mmap_.Remap(hit_buffer_fd_.get(), kHeaderFileOffset, header_size()); header_ = std::make_unique<LiteIndex_HeaderImpl>( reinterpret_cast<LiteIndex_HeaderImpl::HeaderData*>( header_mmap_.address())); @@ -352,6 +357,73 @@ libtextclassifier3::StatusOr<uint32_t> LiteIndex::GetTermId( return tvi; } +void LiteIndex::ScoreAndAppendFetchedHit( + const Hit& hit, SectionIdMask section_id_mask, + bool only_from_prefix_sections, + SuggestionScoringSpecProto::SuggestionRankingStrategy::Code score_by, + const SuggestionResultChecker* suggestion_result_checker, + DocumentId& last_document_id, bool& is_last_document_desired, + int& total_score_out, std::vector<DocHitInfo>* hits_out, + std::vector<Hit::TermFrequencyArray>* term_frequency_out) const { + // Check sections. + if (((UINT64_C(1) << hit.section_id()) & section_id_mask) == 0) { + return; + } + // Check prefix section only. + if (only_from_prefix_sections && !hit.is_in_prefix_section()) { + return; + } + // Check whether this Hit is desired. + // TODO(b/230553264) Move common logic into helper function once we support + // score term by prefix_hit in lite_index. + DocumentId document_id = hit.document_id(); + bool is_new_document = document_id != last_document_id; + if (is_new_document) { + last_document_id = document_id; + is_last_document_desired = + suggestion_result_checker == nullptr || + suggestion_result_checker->BelongsToTargetResults(document_id, + hit.section_id()); + } + if (!is_last_document_desired) { + // The document is removed or expired or not desired. + return; + } + + // Score the hit by the strategy + switch (score_by) { + case SuggestionScoringSpecProto::SuggestionRankingStrategy::NONE: + total_score_out = 1; + break; + case SuggestionScoringSpecProto::SuggestionRankingStrategy::DOCUMENT_COUNT: + if (is_new_document) { + ++total_score_out; + } + break; + case SuggestionScoringSpecProto::SuggestionRankingStrategy::TERM_FREQUENCY: + if (hit.has_term_frequency()) { + total_score_out += hit.term_frequency(); + } else { + ++total_score_out; + } + break; + } + + // Append the Hit or update hit section to the output vector. + if (is_new_document && hits_out != nullptr) { + hits_out->push_back(DocHitInfo(document_id)); + if (term_frequency_out != nullptr) { + term_frequency_out->push_back(Hit::TermFrequencyArray()); + } + } + if (hits_out != nullptr) { + hits_out->back().UpdateSection(hit.section_id()); + if (term_frequency_out != nullptr) { + term_frequency_out->back()[hit.section_id()] = hit.term_frequency(); + } + } +} + int LiteIndex::FetchHits( uint32_t term_id, SectionIdMask section_id_mask, bool only_from_prefix_sections, @@ -359,19 +431,38 @@ int LiteIndex::FetchHits( const SuggestionResultChecker* suggestion_result_checker, std::vector<DocHitInfo>* hits_out, std::vector<Hit::TermFrequencyArray>* term_frequency_out) { - int score = 0; - DocumentId last_document_id = kInvalidDocumentId; - // Record whether the last document belongs to the given namespaces. - bool is_last_document_desired = false; - - if (NeedSort()) { - // Transition from shared_lock in NeedSort to unique_lock here is safe - // because it doesn't hurt to sort again if sorting was done already by - // another thread after NeedSort is evaluated. NeedSort is called before - // sorting to improve concurrency as threads can avoid acquiring the unique - // lock if no sorting is needed. + bool need_sort_at_querying = false; + { + absl_ports::shared_lock l(&mutex_); + + // We sort here when: + // 1. We don't enable sorting at indexing time (i.e. we sort at querying + // time), and there is an unsorted tail portion. OR + // 2. The unsorted tail size exceeds the hit_buffer_sort_threshold, + // regardless of whether or not hit_buffer_sort_at_indexing is enabled. + // This is more of a sanity check. We should not really be encountering + // this case. + need_sort_at_querying = NeedSortAtQuerying(); + } + if (need_sort_at_querying) { absl_ports::unique_lock l(&mutex_); - SortHits(); + IcingTimer timer; + + // Transition from shared_lock to unique_lock is safe here because it + // doesn't hurt to sort again if sorting was done already by another thread + // after need_sort_at_querying is evaluated. + // We check need_sort_at_querying to improve query concurrency as threads + // can avoid acquiring the unique lock if no sorting is needed. + SortHitsImpl(); + + if (options_.hit_buffer_sort_at_indexing) { + // This is the second case for sort. Log as this should be a very rare + // occasion. + ICING_LOG(WARNING) << "Sorting HitBuffer at querying time when " + "hit_buffer_sort_at_indexing is enabled. Sort and " + "merge HitBuffer in " + << timer.Elapsed() * 1000 << " ms."; + } } // This downgrade from an unique_lock to a shared_lock is safe because we're @@ -379,75 +470,72 @@ int LiteIndex::FetchHits( // only in Seek(). // Any operations that might execute in between the transition of downgrading // the lock here are guaranteed not to alter the searchable section (or the - // LiteIndex due to a global lock in IcingSearchEngine). + // LiteIndex) due to a global lock in IcingSearchEngine. absl_ports::shared_lock l(&mutex_); - for (uint32_t idx = Seek(term_id); idx < header_->searchable_end(); idx++) { - TermIdHitPair term_id_hit_pair = - hit_buffer_.array_cast<TermIdHitPair>()[idx]; - if (term_id_hit_pair.term_id() != term_id) break; - - const Hit& hit = term_id_hit_pair.hit(); - // Check sections. - if (((UINT64_C(1) << hit.section_id()) & section_id_mask) == 0) { - continue; - } - // Check prefix section only. - if (only_from_prefix_sections && !hit.is_in_prefix_section()) { - continue; - } - // TODO(b/230553264) Move common logic into helper function once we support - // score term by prefix_hit in lite_index. - // Check whether this Hit is desired. - DocumentId document_id = hit.document_id(); - bool is_new_document = document_id != last_document_id; - if (is_new_document) { - last_document_id = document_id; - is_last_document_desired = - suggestion_result_checker == nullptr || - suggestion_result_checker->BelongsToTargetResults(document_id, - hit.section_id()); - } - if (!is_last_document_desired) { - // The document is removed or expired or not desired. - continue; - } - // Score the hit by the strategy - switch (score_by) { - case SuggestionScoringSpecProto::SuggestionRankingStrategy::NONE: - score = 1; - break; - case SuggestionScoringSpecProto::SuggestionRankingStrategy:: - DOCUMENT_COUNT: - if (is_new_document) { - ++score; - } - break; - case SuggestionScoringSpecProto::SuggestionRankingStrategy:: - TERM_FREQUENCY: - if (hit.has_term_frequency()) { - score += hit.term_frequency(); - } else { - ++score; - } - break; - } + // Search in the HitBuffer array for Hits with the corresponding term_id. + // Hits are added in increasing order of doc ids, so hits that get appended + // later have larger docIds. This means that: + // 1. Hits in the unsorted tail will have larger docIds than hits in the + // sorted portion. + // 2. Hits at the end of the unsorted tail will have larger docIds than hits + // in the front of the tail. + // We want to retrieve hits in descending order of docIds. Therefore we should + // search by doing: + // 1. Linear search first in reverse iteration order over the unsorted tail + // portion. + // 2. Followed by binary search on the sorted portion. + const TermIdHitPair* array = hit_buffer_.array_cast<TermIdHitPair>(); - // Append the Hit or update hit section to the output vector. - if (is_new_document && hits_out != nullptr) { - hits_out->push_back(DocHitInfo(document_id)); - if (term_frequency_out != nullptr) { - term_frequency_out->push_back(Hit::TermFrequencyArray()); + DocumentId last_document_id = kInvalidDocumentId; + // Record whether the last document belongs to the given namespaces. + bool is_last_document_desired = false; + int total_score = 0; + + // Linear search over unsorted tail in reverse iteration order. + // This should only be performed when hit_buffer_sort_at_indexing is enabled. + // When disabled, the entire HitBuffer should be sorted already and only + // binary search is needed. + if (options_.hit_buffer_sort_at_indexing) { + uint32_t unsorted_length = header_->cur_size() - header_->searchable_end(); + for (uint32_t i = 1; i <= unsorted_length; ++i) { + TermIdHitPair term_id_hit_pair = array[header_->cur_size() - i]; + if (term_id_hit_pair.term_id() == term_id) { + // We've found a matched hit. + const Hit& matched_hit = term_id_hit_pair.hit(); + // Score the hit and add to total_score. Also add the hits and its term + // frequency info to hits_out and term_frequency_out if the two vectors + // are non-null. + ScoreAndAppendFetchedHit(matched_hit, section_id_mask, + only_from_prefix_sections, score_by, + suggestion_result_checker, last_document_id, + is_last_document_desired, total_score, + hits_out, term_frequency_out); } } - if (hits_out != nullptr) { - hits_out->back().UpdateSection(hit.section_id()); - if (term_frequency_out != nullptr) { - term_frequency_out->back()[hit.section_id()] = hit.term_frequency(); - } + } + + // Do binary search over the sorted section and repeat the above steps. + TermIdHitPair target_term_id_hit_pair( + term_id, Hit(Hit::kMaxDocumentIdSortValue, Hit::kDefaultTermFrequency)); + for (const TermIdHitPair* ptr = std::lower_bound( + array, array + header_->searchable_end(), target_term_id_hit_pair); + ptr < array + header_->searchable_end(); ++ptr) { + if (ptr->term_id() != term_id) { + // We've processed all matches. Stop iterating further. + break; } + + const Hit& matched_hit = ptr->hit(); + // Score the hit and add to total_score. Also add the hits and its term + // frequency info to hits_out and term_frequency_out if the two vectors are + // non-null. + ScoreAndAppendFetchedHit( + matched_hit, section_id_mask, only_from_prefix_sections, score_by, + suggestion_result_checker, last_document_id, is_last_document_desired, + total_score, hits_out, term_frequency_out); } - return score; + return total_score; } libtextclassifier3::StatusOr<int> LiteIndex::ScoreHits( @@ -455,9 +543,9 @@ libtextclassifier3::StatusOr<int> LiteIndex::ScoreHits( SuggestionScoringSpecProto::SuggestionRankingStrategy::Code score_by, const SuggestionResultChecker* suggestion_result_checker) { return FetchHits(term_id, kSectionIdMaskAll, - /*only_from_prefix_sections=*/false, score_by, - suggestion_result_checker, - /*hits_out=*/nullptr); + /*only_from_prefix_sections=*/false, score_by, + suggestion_result_checker, + /*hits_out=*/nullptr); } bool LiteIndex::is_full() const { @@ -515,7 +603,7 @@ IndexStorageInfoProto LiteIndex::GetStorageInfo( return storage_info; } -void LiteIndex::SortHits() { +void LiteIndex::SortHitsImpl() { // Make searchable by sorting by hit buffer. uint32_t sort_len = header_->cur_size() - header_->searchable_end(); if (sort_len <= 0) { @@ -546,25 +634,6 @@ void LiteIndex::SortHits() { UpdateChecksum(); } -uint32_t LiteIndex::Seek(uint32_t term_id) const { - // Binary search for our term_id. Make sure we get the first - // element. Using kBeginSortValue ensures this for the hit value. - TermIdHitPair term_id_hit_pair( - term_id, Hit(Hit::kMaxDocumentIdSortValue, Hit::kDefaultTermFrequency)); - - const TermIdHitPair::Value* array = - hit_buffer_.array_cast<TermIdHitPair::Value>(); - if (header_->searchable_end() != header_->cur_size()) { - ICING_LOG(WARNING) << "Lite index: hit buffer searchable end != current " - << "size during Seek(): " - << header_->searchable_end() << " vs " - << header_->cur_size(); - } - const TermIdHitPair::Value* ptr = std::lower_bound( - array, array + header_->searchable_end(), term_id_hit_pair.value()); - return ptr - array; -} - libtextclassifier3::Status LiteIndex::Optimize( const std::vector<DocumentId>& document_id_old_to_new, const TermIdCodec* term_id_codec, DocumentId new_last_added_document_id) { @@ -575,7 +644,7 @@ libtextclassifier3::Status LiteIndex::Optimize( } // Sort the hits so that hits with the same term id will be grouped together, // which helps later to determine which terms will be unused after compaction. - SortHits(); + SortHitsImpl(); uint32_t new_size = 0; uint32_t curr_term_id = 0; uint32_t curr_tvi = 0; diff --git a/icing/index/lite/lite-index.h b/icing/index/lite/lite-index.h index 916a14b..288602a 100644 --- a/icing/index/lite/lite-index.h +++ b/icing/index/lite/lite-index.h @@ -20,6 +20,7 @@ #define ICING_INDEX_LITE_INDEX_H_ #include <cstdint> +#include <iterator> #include <limits> #include <memory> #include <string> @@ -48,7 +49,6 @@ #include "icing/store/document-id.h" #include "icing/store/namespace-id.h" #include "icing/store/suggestion-result-checker.h" -#include "icing/util/bit-util.h" #include "icing/util/crc32.h" namespace icing { @@ -63,6 +63,9 @@ class LiteIndex { // An entry in the hit buffer. using Options = LiteIndexOptions; + // Offset for the LiteIndex_Header in the hit buffer mmap. + static constexpr uint32_t kHeaderFileOffset = 0; + // Updates checksum of subcomponents. ~LiteIndex(); @@ -152,8 +155,8 @@ class LiteIndex { // Add all hits with term_id from the sections specified in section_id_mask, // skipping hits in non-prefix sections if only_from_prefix_sections is true, // to hits_out. If hits_out is nullptr, no hits will be added. The - // corresponding hit term frequencies will also be added if term_frequency_out - // is nullptr. + // corresponding hit term frequencies will also not be added if + // term_frequency_out is nullptr. // // Only those hits which belongs to the given namespaces will be counted and // fetched. A nullptr namespace checker will disable this check. @@ -181,15 +184,29 @@ class LiteIndex { uint32_t size() const ICING_LOCKS_EXCLUDED(mutex_) { absl_ports::shared_lock l(&mutex_); - return sizeLocked(); + return size_impl(); } bool WantsMerge() const ICING_LOCKS_EXCLUDED(mutex_) { absl_ports::shared_lock l(&mutex_); - return is_full() || sizeLocked() >= (options_.hit_buffer_want_merge_bytes / - sizeof(TermIdHitPair::Value)); + return is_full() || size_impl() >= (options_.hit_buffer_want_merge_bytes / + sizeof(TermIdHitPair::Value)); + } + + // Whether or not the HitBuffer's unsorted tail size exceeds the sort + // threshold. + bool HasUnsortedHitsExceedingSortThreshold() const + ICING_LOCKS_EXCLUDED(mutex_) { + absl_ports::shared_lock l(&mutex_); + return HasUnsortedHitsExceedingSortThresholdImpl(); } + // Sort hits stored in the index. + void SortHits() ICING_LOCKS_EXCLUDED(mutex_) { + absl_ports::unique_lock l(&mutex_); + SortHitsImpl(); + }; + class const_iterator { friend class LiteIndex; @@ -326,17 +343,13 @@ class LiteIndex { // Check if the hit buffer has reached its capacity. bool is_full() const ICING_SHARED_LOCKS_REQUIRED(mutex_); - uint32_t sizeLocked() const ICING_SHARED_LOCKS_REQUIRED(mutex_) { - return header_->cur_size(); - } - // Non-locking implementation for empty(). bool empty_impl() const ICING_SHARED_LOCKS_REQUIRED(mutex_) { return size_impl() == 0; } // Non-locking implementation for size(). - bool size_impl() const ICING_SHARED_LOCKS_REQUIRED(mutex_) { + uint32_t size_impl() const ICING_SHARED_LOCKS_REQUIRED(mutex_) { return header_->cur_size(); } @@ -352,18 +365,48 @@ class LiteIndex { NamespaceId namespace_id) ICING_EXCLUSIVE_LOCKS_REQUIRED(mutex_); - // Whether or not the HitBuffer requires sorting. - bool NeedSort() ICING_LOCKS_EXCLUDED(mutex_) { - absl_ports::shared_lock l(&mutex_); - return header_->cur_size() - header_->searchable_end() > 0; + // We need to sort during querying time when: + // 1. Sorting at indexing time is not enabled and there is an unsorted tail + // section in the HitBuffer. + // 2. The unsorted tail size exceeds the hit_buffer_sort_threshold, regardless + // of whether or not hit_buffer_sort_at_indexing is enabled. This is to + // prevent performing sequential search on a large unsorted tail section, + // which would result in bad query performance. + // This is more of a sanity check. We should not really be encountering + // this case. + bool NeedSortAtQuerying() const ICING_SHARED_LOCKS_REQUIRED(mutex_) { + return HasUnsortedHitsExceedingSortThresholdImpl() || + (!options_.hit_buffer_sort_at_indexing && + header_->cur_size() - header_->searchable_end() > 0); } - // Sort hits stored in the index. - void SortHits() ICING_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Non-locking implementation for HasUnsortedHitsExceedingSortThresholdImpl(). + bool HasUnsortedHitsExceedingSortThresholdImpl() const + ICING_SHARED_LOCKS_REQUIRED(mutex_) { + return header_->cur_size() - header_->searchable_end() >= + (options_.hit_buffer_sort_threshold_bytes / + sizeof(TermIdHitPair::Value)); + } - // Returns the position of the first element with term_id, or the searchable - // end of the hit buffer if term_id is not present. - uint32_t Seek(uint32_t term_id) const ICING_SHARED_LOCKS_REQUIRED(mutex_); + // Non-locking implementation for SortHits(). + void SortHitsImpl() ICING_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + + // Calculates and adds the score for a fetched hit to total_score_out, while + // updating last_document_id (which keeps track of the last added docId so + // far), and is_last_document_desired (which keeps track of whether that last + // added docId belongs to the query's desired namespace.) + // + // Also appends the hit to hits_out and term_frequency_out if the vectors are + // not null. + void ScoreAndAppendFetchedHit( + const Hit& hit, SectionIdMask section_id_mask, + bool only_from_prefix_sections, + SuggestionScoringSpecProto::SuggestionRankingStrategy::Code score_by, + const SuggestionResultChecker* suggestion_result_checker, + DocumentId& last_document_id, bool& is_last_document_desired, + int& total_score_out, std::vector<DocHitInfo>* hits_out, + std::vector<Hit::TermFrequencyArray>* term_frequency_out) const + ICING_SHARED_LOCKS_REQUIRED(mutex_); // File descriptor that points to where the header and hit buffer are written // to. diff --git a/icing/index/lite/lite-index_test.cc b/icing/index/lite/lite-index_test.cc index 5f141ed..9811fa2 100644 --- a/icing/index/lite/lite-index_test.cc +++ b/icing/index/lite/lite-index_test.cc @@ -14,14 +14,27 @@ #include "icing/index/lite/lite-index.h" +#include <cstdint> +#include <memory> +#include <string> +#include <unordered_map> #include <vector> #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "icing/file/filesystem.h" +#include "icing/index/hit/doc-hit-info.h" +#include "icing/index/hit/hit.h" +#include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/index/lite/doc-hit-info-iterator-term-lite.h" +#include "icing/index/lite/lite-index-header.h" #include "icing/index/term-id-codec.h" +#include "icing/legacy/index/icing-dynamic-trie.h" +#include "icing/legacy/index/icing-filesystem.h" +#include "icing/proto/scoring.pb.h" +#include "icing/proto/term.pb.h" #include "icing/schema/section.h" -#include "icing/store/suggestion-result-checker.h" +#include "icing/store/namespace-id.h" #include "icing/testing/always-false-suggestion-result-checker-impl.h" #include "icing/testing/common-matchers.h" #include "icing/testing/tmp-directory.h" @@ -34,6 +47,8 @@ namespace { using ::testing::ElementsAre; using ::testing::Eq; using ::testing::IsEmpty; +using ::testing::IsFalse; +using ::testing::IsTrue; using ::testing::SizeIs; class LiteIndexTest : public testing::Test { @@ -41,62 +56,518 @@ class LiteIndexTest : public testing::Test { void SetUp() override { index_dir_ = GetTestTempDir() + "/test_dir"; ASSERT_TRUE(filesystem_.CreateDirectoryRecursively(index_dir_.c_str())); - - std::string lite_index_file_name = index_dir_ + "/test_file.lite-idx.index"; - LiteIndex::Options options(lite_index_file_name, - /*hit_buffer_want_merge_bytes=*/1024 * 1024); - ICING_ASSERT_OK_AND_ASSIGN(lite_index_, - LiteIndex::Create(options, &icing_filesystem_)); - - ICING_ASSERT_OK_AND_ASSIGN( - term_id_codec_, - TermIdCodec::Create( - IcingDynamicTrie::max_value_index(IcingDynamicTrie::Options()), - IcingDynamicTrie::max_value_index(options.lexicon_options))); } void TearDown() override { term_id_codec_.reset(); - lite_index_.reset(); ASSERT_TRUE(filesystem_.DeleteDirectoryRecursively(index_dir_.c_str())); } std::string index_dir_; Filesystem filesystem_; IcingFilesystem icing_filesystem_; - std::unique_ptr<LiteIndex> lite_index_; std::unique_ptr<TermIdCodec> term_id_codec_; }; constexpr NamespaceId kNamespace0 = 0; -TEST_F(LiteIndexTest, LiteIndexAppendHits) { +TEST_F(LiteIndexTest, + LiteIndexFetchHits_sortAtQuerying_unsortedHitsBelowSortThreshold) { + // Set up LiteIndex and TermIdCodec + std::string lite_index_file_name = index_dir_ + "/test_file.lite-idx.index"; + // At 64 bytes the unsorted tail can contain a max of 8 TermHitPairs. + LiteIndex::Options options(lite_index_file_name, + /*hit_buffer_want_merge_bytes=*/1024 * 1024, + /*hit_buffer_sort_at_indexing=*/false, + /*hit_buffer_sort_threshold_bytes=*/64); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LiteIndex> lite_index, + LiteIndex::Create(options, &icing_filesystem_)); ICING_ASSERT_OK_AND_ASSIGN( - uint32_t tvi, - lite_index_->InsertTerm("foo", TermMatchType::PREFIX, kNamespace0)); + term_id_codec_, + TermIdCodec::Create( + IcingDynamicTrie::max_value_index(IcingDynamicTrie::Options()), + IcingDynamicTrie::max_value_index(options.lexicon_options))); + + // Add some hits + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t foo_tvi, + lite_index->InsertTerm("foo", TermMatchType::PREFIX, kNamespace0)); ICING_ASSERT_OK_AND_ASSIGN(uint32_t foo_term_id, - term_id_codec_->EncodeTvi(tvi, TviType::LITE)); - Hit doc_hit0(/*section_id=*/0, /*document_id=*/0, Hit::kDefaultTermFrequency, + term_id_codec_->EncodeTvi(foo_tvi, TviType::LITE)); + Hit foo_hit0(/*section_id=*/0, /*document_id=*/1, Hit::kDefaultTermFrequency, /*is_in_prefix_section=*/false); - Hit doc_hit1(/*section_id=*/1, /*document_id=*/0, Hit::kDefaultTermFrequency, + Hit foo_hit1(/*section_id=*/1, /*document_id=*/1, Hit::kDefaultTermFrequency, /*is_in_prefix_section=*/false); - ICING_ASSERT_OK(lite_index_->AddHit(foo_term_id, doc_hit0)); - ICING_ASSERT_OK(lite_index_->AddHit(foo_term_id, doc_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, foo_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, foo_hit1)); + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t bar_tvi, + lite_index->InsertTerm("bar", TermMatchType::PREFIX, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t bar_term_id, + term_id_codec_->EncodeTvi(bar_tvi, TviType::LITE)); + Hit bar_hit0(/*section_id=*/0, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit bar_hit1(/*section_id=*/1, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, bar_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, bar_hit1)); + + // Check that unsorted hits does not exceed the sort threshold. + EXPECT_THAT(lite_index->HasUnsortedHitsExceedingSortThreshold(), IsFalse()); + + // Check that hits are unsorted. Persist the data and pread from + // LiteIndexHeader. + ASSERT_THAT(lite_index->PersistToDisk(), IsOk()); + LiteIndex_HeaderImpl::HeaderData header_data; + ASSERT_TRUE(filesystem_.PRead((lite_index_file_name + "hb").c_str(), + &header_data, sizeof(header_data), + LiteIndex::kHeaderFileOffset)); + EXPECT_THAT(header_data.cur_size - header_data.searchable_end, Eq(4)); + + // Query the LiteIndex std::vector<DocHitInfo> hits1; - lite_index_->FetchHits( + lite_index->FetchHits( foo_term_id, kSectionIdMaskAll, /*only_from_prefix_sections=*/false, SuggestionScoringSpecProto::SuggestionRankingStrategy::DOCUMENT_COUNT, /*namespace_checker=*/nullptr, &hits1); EXPECT_THAT(hits1, SizeIs(1)); - EXPECT_THAT(hits1.back().document_id(), Eq(0)); + EXPECT_THAT(hits1.back().document_id(), Eq(1)); // Check that the hits are coming from section 0 and section 1. EXPECT_THAT(hits1.back().hit_section_ids_mask(), Eq(0b11)); std::vector<DocHitInfo> hits2; AlwaysFalseSuggestionResultCheckerImpl always_false_suggestion_result_checker; - lite_index_->FetchHits( + lite_index->FetchHits( + foo_term_id, kSectionIdMaskAll, + /*only_from_prefix_sections=*/false, + SuggestionScoringSpecProto::SuggestionRankingStrategy::DOCUMENT_COUNT, + &always_false_suggestion_result_checker, &hits2); + // Check that no hits are returned because they get skipped by the namespace + // checker. + EXPECT_THAT(hits2, IsEmpty()); + + // Check that hits are sorted after querying LiteIndex. Persist the data and + // pread from LiteIndexHeader. + ASSERT_THAT(lite_index->PersistToDisk(), IsOk()); + ASSERT_TRUE(filesystem_.PRead((lite_index_file_name + "hb").c_str(), + &header_data, sizeof(header_data), + LiteIndex::kHeaderFileOffset)); + EXPECT_THAT(header_data.cur_size - header_data.searchable_end, Eq(0)); +} + +TEST_F(LiteIndexTest, + LiteIndexFetchHits_sortAtIndexing_unsortedHitsBelowSortThreshold) { + // Set up LiteIndex and TermIdCodec + std::string lite_index_file_name = index_dir_ + "/test_file.lite-idx.index"; + // At 64 bytes the unsorted tail can contain a max of 8 TermHitPairs. + // However note that in these tests we're unable to sort hits after + // indexing, as sorting performed by the string-section-indexing-handler + // after indexing all hits in an entire document, rather than after each + // AddHits() operation. + LiteIndex::Options options(lite_index_file_name, + /*hit_buffer_want_merge_bytes=*/1024 * 1024, + /*hit_buffer_sort_at_indexing=*/true, + /*hit_buffer_sort_threshold_bytes=*/64); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LiteIndex> lite_index, + LiteIndex::Create(options, &icing_filesystem_)); + ICING_ASSERT_OK_AND_ASSIGN( + term_id_codec_, + TermIdCodec::Create( + IcingDynamicTrie::max_value_index(IcingDynamicTrie::Options()), + IcingDynamicTrie::max_value_index(options.lexicon_options))); + + // Add some hits + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t foo_tvi, + lite_index->InsertTerm("foo", TermMatchType::PREFIX, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t foo_term_id, + term_id_codec_->EncodeTvi(foo_tvi, TviType::LITE)); + Hit foo_hit0(/*section_id=*/0, /*document_id=*/1, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit foo_hit1(/*section_id=*/1, /*document_id=*/1, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, foo_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, foo_hit1)); + + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t bar_tvi, + lite_index->InsertTerm("bar", TermMatchType::PREFIX, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t bar_term_id, + term_id_codec_->EncodeTvi(bar_tvi, TviType::LITE)); + Hit bar_hit0(/*section_id=*/0, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit bar_hit1(/*section_id=*/1, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, bar_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, bar_hit1)); + + // Check that unsorted hits does not exceed the sort threshold. + EXPECT_THAT(lite_index->HasUnsortedHitsExceedingSortThreshold(), IsFalse()); + + // Check that hits are unsorted. Persist the data and pread from + // LiteIndexHeader. + ASSERT_THAT(lite_index->PersistToDisk(), IsOk()); + LiteIndex_HeaderImpl::HeaderData header_data; + ASSERT_TRUE(filesystem_.PRead((lite_index_file_name + "hb").c_str(), + &header_data, sizeof(header_data), + LiteIndex::kHeaderFileOffset)); + EXPECT_THAT(header_data.cur_size - header_data.searchable_end, Eq(4)); + + // Query the LiteIndex + std::vector<DocHitInfo> hits1; + lite_index->FetchHits( + foo_term_id, kSectionIdMaskAll, + /*only_from_prefix_sections=*/false, + SuggestionScoringSpecProto::SuggestionRankingStrategy::DOCUMENT_COUNT, + /*namespace_checker=*/nullptr, &hits1); + EXPECT_THAT(hits1, SizeIs(1)); + EXPECT_THAT(hits1.back().document_id(), Eq(1)); + // Check that the hits are coming from section 0 and section 1. + EXPECT_THAT(hits1.back().hit_section_ids_mask(), Eq(0b11)); + + std::vector<DocHitInfo> hits2; + AlwaysFalseSuggestionResultCheckerImpl always_false_suggestion_result_checker; + lite_index->FetchHits( + foo_term_id, kSectionIdMaskAll, + /*only_from_prefix_sections=*/false, + SuggestionScoringSpecProto::SuggestionRankingStrategy::DOCUMENT_COUNT, + &always_false_suggestion_result_checker, &hits2); + // Check that no hits are returned because they get skipped by the namespace + // checker. + EXPECT_THAT(hits2, IsEmpty()); + + // Check that hits are still unsorted after querying LiteIndex because the + // HitBuffer unsorted size is still below the sort threshold, and we've + // enabled sort_at_indexing. + // Persist the data and performing a pread on LiteIndexHeader. + ASSERT_THAT(lite_index->PersistToDisk(), IsOk()); + ASSERT_TRUE(filesystem_.PRead((lite_index_file_name + "hb").c_str(), + &header_data, sizeof(header_data), + LiteIndex::kHeaderFileOffset)); + EXPECT_THAT(header_data.cur_size - header_data.searchable_end, Eq(4)); +} + +TEST_F( + LiteIndexTest, + LiteIndexFetchHits_sortAtQuerying_unsortedHitsExceedingSortAtIndexThreshold) { + // Set up LiteIndex and TermIdCodec + std::string lite_index_file_name = index_dir_ + "/test_file.lite-idx.index"; + // At 64 bytes the unsorted tail can contain a max of 8 TermHitPairs. + // However note that in these tests we're unable to sort hits after + // indexing, as sorting performed by the string-section-indexing-handler + // after indexing all hits in an entire document, rather than after each + // AddHits() operation. + LiteIndex::Options options(lite_index_file_name, + /*hit_buffer_want_merge_bytes=*/1024 * 1024, + /*hit_buffer_sort_at_indexing=*/false, + /*hit_buffer_sort_threshold_bytes=*/64); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LiteIndex> lite_index, + LiteIndex::Create(options, &icing_filesystem_)); + ICING_ASSERT_OK_AND_ASSIGN( + term_id_codec_, + TermIdCodec::Create( + IcingDynamicTrie::max_value_index(IcingDynamicTrie::Options()), + IcingDynamicTrie::max_value_index(options.lexicon_options))); + + // Create 4 hits for docs 0-2, and 2 hits for doc 3 -- 14 in total + // Doc 0 + Hit doc0_hit0(/*section_id=*/0, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc0_hit1(/*section_id=*/0, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc0_hit2(/*section_id=*/1, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc0_hit3(/*section_id=*/2, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + // Doc 1 + Hit doc1_hit0(/*section_id=*/0, /*document_id=*/1, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc1_hit1(/*section_id=*/0, /*document_id=*/1, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc1_hit2(/*section_id=*/1, /*document_id=*/1, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc1_hit3(/*section_id=*/2, /*document_id=*/1, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + // Doc 2 + Hit doc2_hit0(/*section_id=*/0, /*document_id=*/2, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc2_hit1(/*section_id=*/0, /*document_id=*/2, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc2_hit2(/*section_id=*/1, /*document_id=*/2, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc2_hit3(/*section_id=*/2, /*document_id=*/2, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + // Doc 3 + Hit doc3_hit0(/*section_id=*/0, /*document_id=*/3, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc3_hit1(/*section_id=*/0, /*document_id=*/3, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + + // Create terms + // Foo + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t foo_tvi, + lite_index->InsertTerm("foo", TermMatchType::EXACT_ONLY, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t foo_term_id, + term_id_codec_->EncodeTvi(foo_tvi, TviType::LITE)); + // Bar + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t bar_tvi, + lite_index->InsertTerm("bar", TermMatchType::PREFIX, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t bar_term_id, + term_id_codec_->EncodeTvi(bar_tvi, TviType::LITE)); + // Baz + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t baz_tvi, + lite_index->InsertTerm("baz", TermMatchType::PREFIX, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t baz_term_id, + term_id_codec_->EncodeTvi(baz_tvi, TviType::LITE)); + // Qux + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t qux_tvi, + lite_index->InsertTerm("qux", TermMatchType::PREFIX, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t qux_term_id, + term_id_codec_->EncodeTvi(qux_tvi, TviType::LITE)); + + // Add 14 hits and make sure that termIds are added in unsorted order. + // Documents should be inserted in order as new incoming hits should have + // larger document ids. + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc0_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, doc0_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(baz_term_id, doc0_hit2)); + ICING_ASSERT_OK(lite_index->AddHit(qux_term_id, doc0_hit3)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc1_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, doc1_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc1_hit2)); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, doc1_hit3)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc2_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(baz_term_id, doc2_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(qux_term_id, doc2_hit2)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc2_hit3)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc3_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(baz_term_id, doc3_hit1)); + // Verify that the HitBuffer has not been sorted. + EXPECT_THAT(lite_index->HasUnsortedHitsExceedingSortThreshold(), IsTrue()); + + // We now have the following in the hit buffer: + // <term>: {(docId, sectionId)...} + // foo: {(0, 0); (1, 0); (1, 1); (2, 0); (2, 2); (3, 0)} + // bar: {(0, 0); (1, 0); (1, 2)} + // baz: {(0, 1); (2, 0); (3, 0)} + // quz: {(0, 2); (2, 1)} + + // Search over the HitBuffer. + std::vector<DocHitInfo> hits1; + lite_index->FetchHits( + foo_term_id, kSectionIdMaskAll, + /*only_from_prefix_sections=*/false, + SuggestionScoringSpecProto::SuggestionRankingStrategy::DOCUMENT_COUNT, + /*namespace_checker=*/nullptr, &hits1); + EXPECT_THAT(hits1, SizeIs(4)); + // Check that hits are retrieved in descending order of docIds. + EXPECT_THAT(hits1[0].document_id(), Eq(3)); + EXPECT_THAT(hits1[0].hit_section_ids_mask(), Eq(0b1)); + EXPECT_THAT(hits1[1].document_id(), Eq(2)); + EXPECT_THAT(hits1[1].hit_section_ids_mask(), Eq(0b101)); + EXPECT_THAT(hits1[2].document_id(), Eq(1)); + EXPECT_THAT(hits1[2].hit_section_ids_mask(), Eq(0b11)); + EXPECT_THAT(hits1[3].document_id(), Eq(0)); + EXPECT_THAT(hits1[3].hit_section_ids_mask(), Eq(0b1)); + + std::vector<DocHitInfo> hits2; + AlwaysFalseSuggestionResultCheckerImpl always_false_suggestion_result_checker; + lite_index->FetchHits( + foo_term_id, kSectionIdMaskAll, + /*only_from_prefix_sections=*/false, + SuggestionScoringSpecProto::SuggestionRankingStrategy::DOCUMENT_COUNT, + &always_false_suggestion_result_checker, &hits2); + // Check that no hits are returned because they get skipped by the namespace + // checker. + EXPECT_THAT(hits2, IsEmpty()); + + std::vector<DocHitInfo> hits3; + lite_index->FetchHits( + bar_term_id, 0b1, + /*only_from_prefix_sections=*/false, + SuggestionScoringSpecProto::SuggestionRankingStrategy::DOCUMENT_COUNT, + /*namespace_checker=*/nullptr, &hits3); + EXPECT_THAT(hits3, SizeIs(2)); + // Check fetching hits with SectionIdMask. + EXPECT_THAT(hits3[0].document_id(), Eq(1)); + EXPECT_THAT(hits3[1].hit_section_ids_mask(), Eq(0b1)); + EXPECT_THAT(hits3[1].document_id(), Eq(0)); + EXPECT_THAT(hits3[1].hit_section_ids_mask(), Eq(0b1)); + + // Check that the HitBuffer is sorted after the query call. + EXPECT_THAT(lite_index->HasUnsortedHitsExceedingSortThreshold(), IsFalse()); +} + +TEST_F( + LiteIndexTest, + LiteIndexFetchHits_sortAtIndexing_unsortedHitsExceedingSortAtIndexThreshold) { + // Set up LiteIndex and TermIdCodec + std::string lite_index_file_name = index_dir_ + "/test_file.lite-idx.index"; + // At 64 bytes the unsorted tail can contain a max of 8 TermHitPairs. + LiteIndex::Options options(lite_index_file_name, + /*hit_buffer_want_merge_bytes=*/1024 * 1024, + /*hit_buffer_sort_at_indexing=*/true, + /*hit_buffer_sort_threshold_bytes=*/64); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LiteIndex> lite_index, + LiteIndex::Create(options, &icing_filesystem_)); + ICING_ASSERT_OK_AND_ASSIGN( + term_id_codec_, + TermIdCodec::Create( + IcingDynamicTrie::max_value_index(IcingDynamicTrie::Options()), + IcingDynamicTrie::max_value_index(options.lexicon_options))); + + // Create 4 hits for docs 0-2, and 2 hits for doc 3 -- 14 in total + // Doc 0 + Hit doc0_hit0(/*section_id=*/0, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc0_hit1(/*section_id=*/0, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc0_hit2(/*section_id=*/1, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc0_hit3(/*section_id=*/2, /*document_id=*/0, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + // Doc 1 + Hit doc1_hit0(/*section_id=*/0, /*document_id=*/1, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc1_hit1(/*section_id=*/0, /*document_id=*/1, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc1_hit2(/*section_id=*/1, /*document_id=*/1, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc1_hit3(/*section_id=*/2, /*document_id=*/1, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + // Doc 2 + Hit doc2_hit0(/*section_id=*/0, /*document_id=*/2, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc2_hit1(/*section_id=*/0, /*document_id=*/2, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc2_hit2(/*section_id=*/1, /*document_id=*/2, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc2_hit3(/*section_id=*/2, /*document_id=*/2, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + // Doc 3 + Hit doc3_hit0(/*section_id=*/0, /*document_id=*/3, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc3_hit1(/*section_id=*/0, /*document_id=*/3, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc3_hit2(/*section_id=*/1, /*document_id=*/3, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc3_hit3(/*section_id=*/2, /*document_id=*/3, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + // Doc 4 + Hit doc4_hit0(/*section_id=*/0, /*document_id=*/4, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc4_hit1(/*section_id=*/0, /*document_id=*/4, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc4_hit2(/*section_id=*/1, /*document_id=*/4, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + Hit doc4_hit3(/*section_id=*/2, /*document_id=*/4, Hit::kDefaultTermFrequency, + /*is_in_prefix_section=*/false); + + // Create terms + // Foo + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t foo_tvi, + lite_index->InsertTerm("foo", TermMatchType::EXACT_ONLY, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t foo_term_id, + term_id_codec_->EncodeTvi(foo_tvi, TviType::LITE)); + // Bar + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t bar_tvi, + lite_index->InsertTerm("bar", TermMatchType::PREFIX, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t bar_term_id, + term_id_codec_->EncodeTvi(bar_tvi, TviType::LITE)); + // Baz + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t baz_tvi, + lite_index->InsertTerm("baz", TermMatchType::PREFIX, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t baz_term_id, + term_id_codec_->EncodeTvi(baz_tvi, TviType::LITE)); + // Qux + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t qux_tvi, + lite_index->InsertTerm("qux", TermMatchType::PREFIX, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t qux_term_id, + term_id_codec_->EncodeTvi(qux_tvi, TviType::LITE)); + + // Add hits and make sure that termIds are added in unsorted order. + // Documents should be inserted in order as new incoming hits should have + // larger document ids. + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc0_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, doc0_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(baz_term_id, doc0_hit2)); + ICING_ASSERT_OK(lite_index->AddHit(qux_term_id, doc0_hit3)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc1_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, doc1_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc1_hit2)); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, doc1_hit3)); + // Adding 8 hits exceeds the sort threshold. However when sort_at_indexing is + // enabled, sorting is done in the string-section-indexing-handler rather than + // AddHit() itself, we need to invoke SortHits() manually. + EXPECT_THAT(lite_index->HasUnsortedHitsExceedingSortThreshold(), IsTrue()); + lite_index->SortHits(); + // Check that the HitBuffer is sorted. + ASSERT_THAT(lite_index->PersistToDisk(), IsOk()); + LiteIndex_HeaderImpl::HeaderData header_data; + ASSERT_TRUE(filesystem_.PRead((lite_index_file_name + "hb").c_str(), + &header_data, sizeof(header_data), + LiteIndex::kHeaderFileOffset)); + EXPECT_THAT(header_data.cur_size - header_data.searchable_end, Eq(0)); + + // Add 12 more hits so that sort threshold is exceeded again. + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc2_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(baz_term_id, doc2_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(qux_term_id, doc2_hit2)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc2_hit3)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc3_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(baz_term_id, doc3_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc3_hit2)); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, doc3_hit3)); + ICING_ASSERT_OK(lite_index->AddHit(baz_term_id, doc4_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(qux_term_id, doc4_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc4_hit2)); + ICING_ASSERT_OK(lite_index->AddHit(bar_term_id, doc4_hit3)); + + // Adding these hits exceeds the sort threshold. However when sort_at_indexing + // is enabled, sorting is done in the string-section-indexing-handler rather + // than AddHit() itself. + EXPECT_THAT(lite_index->HasUnsortedHitsExceedingSortThreshold(), IsTrue()); + + // We now have the following in the hit buffer: + // <term>: {(docId, sectionId)...} + // foo: {(0, 0); (1, 0); (1, 1); (2, 0); (2, 2); (3, 0); (3, 1); (4, 1)} + // bar: {(0, 0); (1, 0); (1, 2); (3, 2); (4, 2)} + // baz: {(0, 1); (2, 0); (3, 0); (4, 0)} + // quz: {(0, 2); (2, 1); (4, 0)} + + // Search over the HitBuffer. + std::vector<DocHitInfo> hits1; + lite_index->FetchHits( + foo_term_id, kSectionIdMaskAll, + /*only_from_prefix_sections=*/false, + SuggestionScoringSpecProto::SuggestionRankingStrategy::DOCUMENT_COUNT, + /*namespace_checker=*/nullptr, &hits1); + EXPECT_THAT(hits1, SizeIs(5)); + // Check that hits are retrieved in descending order of docIds. + EXPECT_THAT(hits1[0].document_id(), Eq(4)); + EXPECT_THAT(hits1[0].hit_section_ids_mask(), Eq(0b10)); + EXPECT_THAT(hits1[1].document_id(), Eq(3)); + EXPECT_THAT(hits1[1].hit_section_ids_mask(), Eq(0b11)); + EXPECT_THAT(hits1[2].document_id(), Eq(2)); + EXPECT_THAT(hits1[2].hit_section_ids_mask(), Eq(0b101)); + EXPECT_THAT(hits1[3].document_id(), Eq(1)); + EXPECT_THAT(hits1[3].hit_section_ids_mask(), Eq(0b11)); + EXPECT_THAT(hits1[4].document_id(), Eq(0)); + EXPECT_THAT(hits1[4].hit_section_ids_mask(), Eq(0b1)); + + std::vector<DocHitInfo> hits2; + AlwaysFalseSuggestionResultCheckerImpl always_false_suggestion_result_checker; + lite_index->FetchHits( foo_term_id, kSectionIdMaskAll, /*only_from_prefix_sections=*/false, SuggestionScoringSpecProto::SuggestionRankingStrategy::DOCUMENT_COUNT, @@ -104,13 +575,119 @@ TEST_F(LiteIndexTest, LiteIndexAppendHits) { // Check that no hits are returned because they get skipped by the namespace // checker. EXPECT_THAT(hits2, IsEmpty()); + + std::vector<DocHitInfo> hits3; + lite_index->FetchHits( + bar_term_id, 0b1, + /*only_from_prefix_sections=*/false, + SuggestionScoringSpecProto::SuggestionRankingStrategy::DOCUMENT_COUNT, + /*namespace_checker=*/nullptr, &hits3); + EXPECT_THAT(hits3, SizeIs(2)); + // Check fetching hits with SectionIdMask. + EXPECT_THAT(hits3[0].document_id(), Eq(1)); + EXPECT_THAT(hits3[1].hit_section_ids_mask(), Eq(0b1)); + EXPECT_THAT(hits3[1].document_id(), Eq(0)); + EXPECT_THAT(hits3[1].hit_section_ids_mask(), Eq(0b1)); + + // Check that the HitBuffer is sorted after the query call. FetchHits should + // sort before performing binary search if the HitBuffer unsorted size exceeds + // the sort threshold. Regardless of the sort_at_indexing config. + EXPECT_THAT(lite_index->HasUnsortedHitsExceedingSortThreshold(), IsFalse()); + ASSERT_THAT(lite_index->PersistToDisk(), IsOk()); + ASSERT_TRUE(filesystem_.PRead((lite_index_file_name + "hb").c_str(), + &header_data, sizeof(header_data), + LiteIndex::kHeaderFileOffset)); + EXPECT_THAT(header_data.cur_size - header_data.searchable_end, Eq(0)); } TEST_F(LiteIndexTest, LiteIndexIterator) { + // Set up LiteIndex and TermIdCodec + std::string lite_index_file_name = index_dir_ + "/test_file.lite-idx.index"; + // At 64 bytes the unsorted tail can contain a max of 8 TermHitPairs. + LiteIndex::Options options(lite_index_file_name, + /*hit_buffer_want_merge_bytes=*/1024 * 1024, + /*hit_buffer_sort_at_indexing=*/true, + /*hit_buffer_sort_threshold_bytes=*/64); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LiteIndex> lite_index, + LiteIndex::Create(options, &icing_filesystem_)); + ICING_ASSERT_OK_AND_ASSIGN( + term_id_codec_, + TermIdCodec::Create( + IcingDynamicTrie::max_value_index(IcingDynamicTrie::Options()), + IcingDynamicTrie::max_value_index(options.lexicon_options))); + + const std::string term = "foo"; + ICING_ASSERT_OK_AND_ASSIGN( + uint32_t tvi, + lite_index->InsertTerm(term, TermMatchType::PREFIX, kNamespace0)); + ICING_ASSERT_OK_AND_ASSIGN(uint32_t foo_term_id, + term_id_codec_->EncodeTvi(tvi, TviType::LITE)); + Hit doc0_hit0(/*section_id=*/0, /*document_id=*/0, /*term_frequency=*/3, + /*is_in_prefix_section=*/false); + Hit doc0_hit1(/*section_id=*/1, /*document_id=*/0, /*term_frequency=*/5, + /*is_in_prefix_section=*/false); + SectionIdMask doc0_section_id_mask = 0b11; + std::unordered_map<SectionId, Hit::TermFrequency> + expected_section_ids_tf_map0 = {{0, 3}, {1, 5}}; + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc0_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc0_hit1)); + + Hit doc1_hit1(/*section_id=*/1, /*document_id=*/1, /*term_frequency=*/7, + /*is_in_prefix_section=*/false); + Hit doc1_hit2(/*section_id=*/2, /*document_id=*/1, /*term_frequency=*/11, + /*is_in_prefix_section=*/false); + SectionIdMask doc1_section_id_mask = 0b110; + std::unordered_map<SectionId, Hit::TermFrequency> + expected_section_ids_tf_map1 = {{1, 7}, {2, 11}}; + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc1_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc1_hit2)); + + std::unique_ptr<DocHitInfoIteratorTermLiteExact> iter = + std::make_unique<DocHitInfoIteratorTermLiteExact>( + term_id_codec_.get(), lite_index.get(), term, /*term_start_index=*/0, + /*unnormalized_term_length=*/0, kSectionIdMaskAll, + /*need_hit_term_frequency=*/true); + + ASSERT_THAT(iter->Advance(), IsOk()); + EXPECT_THAT(iter->doc_hit_info().document_id(), Eq(1)); + EXPECT_THAT(iter->doc_hit_info().hit_section_ids_mask(), + Eq(doc1_section_id_mask)); + + std::vector<TermMatchInfo> matched_terms_stats; + iter->PopulateMatchedTermsStats(&matched_terms_stats); + EXPECT_THAT(matched_terms_stats, ElementsAre(EqualsTermMatchInfo( + term, expected_section_ids_tf_map1))); + + ASSERT_THAT(iter->Advance(), IsOk()); + EXPECT_THAT(iter->doc_hit_info().document_id(), Eq(0)); + EXPECT_THAT(iter->doc_hit_info().hit_section_ids_mask(), + Eq(doc0_section_id_mask)); + matched_terms_stats.clear(); + iter->PopulateMatchedTermsStats(&matched_terms_stats); + EXPECT_THAT(matched_terms_stats, ElementsAre(EqualsTermMatchInfo( + term, expected_section_ids_tf_map0))); +} + +TEST_F(LiteIndexTest, LiteIndexIterator_sortAtIndexingDisabled) { + // Set up LiteIndex and TermIdCodec + std::string lite_index_file_name = index_dir_ + "/test_file.lite-idx.index"; + // At 64 bytes the unsorted tail can contain a max of 8 TermHitPairs. + LiteIndex::Options options(lite_index_file_name, + /*hit_buffer_want_merge_bytes=*/1024 * 1024, + /*hit_buffer_sort_at_indexing=*/false, + /*hit_buffer_sort_threshold_bytes=*/64); + ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LiteIndex> lite_index, + LiteIndex::Create(options, &icing_filesystem_)); + ICING_ASSERT_OK_AND_ASSIGN( + term_id_codec_, + TermIdCodec::Create( + IcingDynamicTrie::max_value_index(IcingDynamicTrie::Options()), + IcingDynamicTrie::max_value_index(options.lexicon_options))); + const std::string term = "foo"; ICING_ASSERT_OK_AND_ASSIGN( uint32_t tvi, - lite_index_->InsertTerm(term, TermMatchType::PREFIX, kNamespace0)); + lite_index->InsertTerm(term, TermMatchType::PREFIX, kNamespace0)); ICING_ASSERT_OK_AND_ASSIGN(uint32_t foo_term_id, term_id_codec_->EncodeTvi(tvi, TviType::LITE)); Hit doc0_hit0(/*section_id=*/0, /*document_id=*/0, /*term_frequency=*/3, @@ -120,8 +697,8 @@ TEST_F(LiteIndexTest, LiteIndexIterator) { SectionIdMask doc0_section_id_mask = 0b11; std::unordered_map<SectionId, Hit::TermFrequency> expected_section_ids_tf_map0 = {{0, 3}, {1, 5}}; - ICING_ASSERT_OK(lite_index_->AddHit(foo_term_id, doc0_hit0)); - ICING_ASSERT_OK(lite_index_->AddHit(foo_term_id, doc0_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc0_hit0)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc0_hit1)); Hit doc1_hit1(/*section_id=*/1, /*document_id=*/1, /*term_frequency=*/7, /*is_in_prefix_section=*/false); @@ -130,12 +707,12 @@ TEST_F(LiteIndexTest, LiteIndexIterator) { SectionIdMask doc1_section_id_mask = 0b110; std::unordered_map<SectionId, Hit::TermFrequency> expected_section_ids_tf_map1 = {{1, 7}, {2, 11}}; - ICING_ASSERT_OK(lite_index_->AddHit(foo_term_id, doc1_hit1)); - ICING_ASSERT_OK(lite_index_->AddHit(foo_term_id, doc1_hit2)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc1_hit1)); + ICING_ASSERT_OK(lite_index->AddHit(foo_term_id, doc1_hit2)); std::unique_ptr<DocHitInfoIteratorTermLiteExact> iter = std::make_unique<DocHitInfoIteratorTermLiteExact>( - term_id_codec_.get(), lite_index_.get(), term, /*term_start_index=*/0, + term_id_codec_.get(), lite_index.get(), term, /*term_start_index=*/0, /*unnormalized_term_length=*/0, kSectionIdMaskAll, /*need_hit_term_frequency=*/true); diff --git a/icing/index/lite/lite-index_thread-safety_test.cc b/icing/index/lite/lite-index_thread-safety_test.cc index 7711f92..53aa6cd 100644 --- a/icing/index/lite/lite-index_thread-safety_test.cc +++ b/icing/index/lite/lite-index_thread-safety_test.cc @@ -19,12 +19,9 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "icing/index/lite/doc-hit-info-iterator-term-lite.h" #include "icing/index/lite/lite-index.h" #include "icing/index/term-id-codec.h" #include "icing/schema/section.h" -#include "icing/store/suggestion-result-checker.h" -#include "icing/testing/always-false-suggestion-result-checker-impl.h" #include "icing/testing/common-matchers.h" #include "icing/testing/tmp-directory.h" @@ -52,7 +49,9 @@ class LiteIndexThreadSafetyTest : public testing::Test { std::string lite_index_file_name = index_dir_ + "/test_file.lite-idx-thread-safety.index"; LiteIndex::Options options(lite_index_file_name, - /*hit_buffer_want_merge_bytes=*/1024 * 1024); + /*hit_buffer_want_merge_bytes=*/1024 * 1024, + /*hit_buffer_sort_at_indexing=*/true, + /*hit_buffer_sort_threshold_bytes=*/64); ICING_ASSERT_OK_AND_ASSIGN(lite_index_, LiteIndex::Create(options, &icing_filesystem_)); diff --git a/icing/index/lite/term-id-hit-pair.h b/icing/index/lite/term-id-hit-pair.h index 61ec502..82bd010 100644 --- a/icing/index/lite/term-id-hit-pair.h +++ b/icing/index/lite/term-id-hit-pair.h @@ -73,6 +73,8 @@ class TermIdHitPair { return value_ == rhs.value_; } + bool operator<(const TermIdHitPair& rhs) const { return value_ < rhs.value_; } + private: Value value_; }; diff --git a/icing/index/main/doc-hit-info-iterator-term-main.cc b/icing/index/main/doc-hit-info-iterator-term-main.cc index 8f0d3f5..5cf6a4c 100644 --- a/icing/index/main/doc-hit-info-iterator-term-main.cc +++ b/icing/index/main/doc-hit-info-iterator-term-main.cc @@ -14,16 +14,20 @@ #include "icing/index/main/doc-hit-info-iterator-term-main.h" -#include <cstdint> #include <memory> +#include <optional> +#include <string> +#include <utility> +#include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" -#include "icing/file/posting_list/posting-list-identifier.h" #include "icing/index/hit/doc-hit-info.h" +#include "icing/index/hit/hit.h" +#include "icing/index/iterator/doc-hit-info-iterator.h" +#include "icing/index/main/main-index.h" #include "icing/index/main/posting-list-hit-accessor.h" -#include "icing/legacy/core/icing-string-util.h" #include "icing/schema/section.h" #include "icing/store/document-id.h" #include "icing/util/logging.h" @@ -44,6 +48,30 @@ std::string SectionIdMaskToString(SectionIdMask section_id_mask) { return mask; } +void MergeNewHitIntoCachedDocHitInfos( + const Hit& hit, bool need_hit_term_frequency, + std::vector<DocHitInfoIteratorTermMain::DocHitInfoAndTermFrequencyArray>& + cached_doc_hit_infos_out) { + if (cached_doc_hit_infos_out.empty() || + hit.document_id() != + cached_doc_hit_infos_out.back().doc_hit_info.document_id()) { + std::optional<Hit::TermFrequencyArray> tf_arr; + if (need_hit_term_frequency) { + tf_arr = std::make_optional<Hit::TermFrequencyArray>(); + } + + cached_doc_hit_infos_out.push_back( + DocHitInfoIteratorTermMain::DocHitInfoAndTermFrequencyArray( + DocHitInfo(hit.document_id()), std::move(tf_arr))); + } + + cached_doc_hit_infos_out.back().doc_hit_info.UpdateSection(hit.section_id()); + if (need_hit_term_frequency) { + (*cached_doc_hit_infos_out.back().term_frequency_array)[hit.section_id()] = + hit.term_frequency(); + } +} + } // namespace libtextclassifier3::Status DocHitInfoIteratorTermMain::Advance() { @@ -76,7 +104,8 @@ libtextclassifier3::Status DocHitInfoIteratorTermMain::Advance() { return absl_ports::ResourceExhaustedError( "No more DocHitInfos in iterator"); } - doc_hit_info_ = cached_doc_hit_infos_.at(cached_doc_hit_infos_idx_); + doc_hit_info_ = + cached_doc_hit_infos_.at(cached_doc_hit_infos_idx_).doc_hit_info; hit_intersect_section_ids_mask_ = doc_hit_info_.hit_section_ids_mask(); return libtextclassifier3::Status::OK; } @@ -90,16 +119,16 @@ DocHitInfoIteratorTermMain::TrimRightMostNode() && { } libtextclassifier3::Status DocHitInfoIteratorTermMainExact::RetrieveMoreHits() { - DocHitInfo last_doc_hit_info; + DocHitInfoAndTermFrequencyArray last_doc_hit_info; if (!cached_doc_hit_infos_.empty()) { - last_doc_hit_info = cached_doc_hit_infos_.back(); + last_doc_hit_info = std::move(cached_doc_hit_infos_.back()); } cached_doc_hit_infos_idx_ = 0; cached_doc_hit_infos_.clear(); - if (last_doc_hit_info.document_id() != kInvalidDocumentId) { + if (last_doc_hit_info.doc_hit_info.document_id() != kInvalidDocumentId) { // Carry over the last hit. It might need to be merged with the first hit of // of the next posting list in the chain. - cached_doc_hit_infos_.push_back(last_doc_hit_info); + cached_doc_hit_infos_.push_back(std::move(last_doc_hit_info)); } if (posting_list_accessor_ == nullptr) { ICING_ASSIGN_OR_RETURN(posting_list_accessor_, @@ -112,8 +141,7 @@ libtextclassifier3::Status DocHitInfoIteratorTermMainExact::RetrieveMoreHits() { all_pages_consumed_ = true; } ++num_blocks_inspected_; - cached_doc_hit_infos_.reserve(hits.size() + 1); - cached_hit_term_frequency_.reserve(hits.size() + 1); + cached_doc_hit_infos_.reserve(cached_doc_hit_infos_.size() + hits.size()); for (const Hit& hit : hits) { // Check sections. if (((UINT64_C(1) << hit.section_id()) & section_restrict_mask_) == 0) { @@ -123,13 +151,9 @@ libtextclassifier3::Status DocHitInfoIteratorTermMainExact::RetrieveMoreHits() { if (hit.is_prefix_hit()) { continue; } - if (cached_doc_hit_infos_.empty() || - hit.document_id() != cached_doc_hit_infos_.back().document_id()) { - cached_doc_hit_infos_.push_back(DocHitInfo(hit.document_id())); - cached_hit_term_frequency_.push_back(Hit::TermFrequencyArray()); - } - cached_doc_hit_infos_.back().UpdateSection(hit.section_id()); - cached_hit_term_frequency_.back()[hit.section_id()] = hit.term_frequency(); + + MergeNewHitIntoCachedDocHitInfos(hit, need_hit_term_frequency_, + cached_doc_hit_infos_); } return libtextclassifier3::Status::OK; } @@ -141,16 +165,16 @@ std::string DocHitInfoIteratorTermMainExact::ToString() const { libtextclassifier3::Status DocHitInfoIteratorTermMainPrefix::RetrieveMoreHits() { - DocHitInfo last_doc_hit_info; + DocHitInfoAndTermFrequencyArray last_doc_hit_info; if (!cached_doc_hit_infos_.empty()) { - last_doc_hit_info = cached_doc_hit_infos_.back(); + last_doc_hit_info = std::move(cached_doc_hit_infos_.back()); } cached_doc_hit_infos_idx_ = 0; cached_doc_hit_infos_.clear(); - if (last_doc_hit_info.document_id() != kInvalidDocumentId) { + if (last_doc_hit_info.doc_hit_info.document_id() != kInvalidDocumentId) { // Carry over the last hit. It might need to be merged with the first hit of // of the next posting list in the chain. - cached_doc_hit_infos_.push_back(last_doc_hit_info); + cached_doc_hit_infos_.push_back(std::move(last_doc_hit_info)); } ++num_blocks_inspected_; @@ -165,10 +189,7 @@ DocHitInfoIteratorTermMainPrefix::RetrieveMoreHits() { if (hits.empty()) { all_pages_consumed_ = true; } - cached_doc_hit_infos_.reserve(hits.size()); - if (need_hit_term_frequency_) { - cached_hit_term_frequency_.reserve(hits.size()); - } + cached_doc_hit_infos_.reserve(cached_doc_hit_infos_.size() + hits.size()); for (const Hit& hit : hits) { // Check sections. if (((UINT64_C(1) << hit.section_id()) & section_restrict_mask_) == 0) { @@ -178,18 +199,9 @@ DocHitInfoIteratorTermMainPrefix::RetrieveMoreHits() { if (!exact_ && !hit.is_in_prefix_section()) { continue; } - if (cached_doc_hit_infos_.empty() || - hit.document_id() != cached_doc_hit_infos_.back().document_id()) { - cached_doc_hit_infos_.push_back(DocHitInfo(hit.document_id())); - if (need_hit_term_frequency_) { - cached_hit_term_frequency_.push_back(Hit::TermFrequencyArray()); - } - } - cached_doc_hit_infos_.back().UpdateSection(hit.section_id()); - if (need_hit_term_frequency_) { - cached_hit_term_frequency_.back()[hit.section_id()] = - hit.term_frequency(); - } + + MergeNewHitIntoCachedDocHitInfos(hit, need_hit_term_frequency_, + cached_doc_hit_infos_); } return libtextclassifier3::Status::OK; } diff --git a/icing/index/main/doc-hit-info-iterator-term-main.h b/icing/index/main/doc-hit-info-iterator-term-main.h index 08a385c..1987e12 100644 --- a/icing/index/main/doc-hit-info-iterator-term-main.h +++ b/icing/index/main/doc-hit-info-iterator-term-main.h @@ -17,10 +17,14 @@ #include <cstdint> #include <memory> +#include <optional> +#include <string> +#include <utility> #include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/index/hit/doc-hit-info.h" +#include "icing/index/hit/hit.h" #include "icing/index/iterator/doc-hit-info-iterator.h" #include "icing/index/main/main-index.h" #include "icing/index/main/posting-list-hit-accessor.h" @@ -31,6 +35,19 @@ namespace lib { class DocHitInfoIteratorTermMain : public DocHitInfoIterator { public: + struct DocHitInfoAndTermFrequencyArray { + DocHitInfo doc_hit_info; + std::optional<Hit::TermFrequencyArray> term_frequency_array; + + explicit DocHitInfoAndTermFrequencyArray() = default; + + explicit DocHitInfoAndTermFrequencyArray( + DocHitInfo doc_hit_info_in, + std::optional<Hit::TermFrequencyArray> term_frequency_array_in) + : doc_hit_info(std::move(doc_hit_info_in)), + term_frequency_array(std::move(term_frequency_array_in)) {} + }; + explicit DocHitInfoIteratorTermMain(MainIndex* main_index, const std::string& term, int term_start_index, @@ -74,8 +91,9 @@ class DocHitInfoIteratorTermMain : public DocHitInfoIterator { while (section_mask_copy) { SectionId section_id = __builtin_ctzll(section_mask_copy); if (need_hit_term_frequency_) { - section_term_frequencies.at(section_id) = cached_hit_term_frequency_.at( - cached_doc_hit_infos_idx_)[section_id]; + section_term_frequencies.at(section_id) = + (*cached_doc_hit_infos_.at(cached_doc_hit_infos_idx_) + .term_frequency_array)[section_id]; } section_mask_copy &= ~(UINT64_C(1) << section_id); } @@ -106,12 +124,13 @@ class DocHitInfoIteratorTermMain : public DocHitInfoIterator { std::unique_ptr<PostingListHitAccessor> posting_list_accessor_; MainIndex* main_index_; - // Stores hits retrieved from the index. This may only be a subset of the hits - // that are present in the index. Current value pointed to by the Iterator is - // tracked by cached_doc_hit_infos_idx_. - std::vector<DocHitInfo> cached_doc_hit_infos_; - std::vector<Hit::TermFrequencyArray> cached_hit_term_frequency_; + // Stores hits and optional term frequency arrays retrieved from the index. + // This may only be a subset of the hits that are present in the index. + // Current value pointed to by the Iterator is tracked by + // cached_doc_hit_infos_idx_. + std::vector<DocHitInfoAndTermFrequencyArray> cached_doc_hit_infos_; int cached_doc_hit_infos_idx_; + int num_advance_calls_; int num_blocks_inspected_; bool all_pages_consumed_; @@ -168,10 +187,6 @@ class DocHitInfoIteratorTermMainPrefix : public DocHitInfoIteratorTermMain { libtextclassifier3::Status RetrieveMoreHits() override; private: - // After retrieving DocHitInfos from the index, a DocHitInfo for docid 1 and - // "foo" and a DocHitInfo for docid 1 and "fool". These DocHitInfos should be - // merged. - void SortAndDedupeDocumentIds(); // Whether or not posting_list_accessor_ holds a posting list chain for // 'term' or for a term for which 'term' is a prefix. This is necessary to // determine whether to return hits that are not from a prefix section (hits diff --git a/icing/index/main/main-index-merger_test.cc b/icing/index/main/main-index-merger_test.cc index 8a2f691..37e14fc 100644 --- a/icing/index/main/main-index-merger_test.cc +++ b/icing/index/main/main-index-merger_test.cc @@ -45,7 +45,9 @@ class MainIndexMergerTest : public testing::Test { std::string lite_index_file_name = index_dir_ + "/test_file.lite-idx.index"; LiteIndex::Options options(lite_index_file_name, - /*hit_buffer_want_merge_bytes=*/1024 * 1024); + /*hit_buffer_want_merge_bytes=*/1024 * 1024, + /*hit_buffer_sort_at_indexing=*/true, + /*hit_buffer_sort_threshold_bytes=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN(lite_index_, LiteIndex::Create(options, &icing_filesystem_)); diff --git a/icing/index/main/main-index.cc b/icing/index/main/main-index.cc index d5e9d57..aae60c6 100644 --- a/icing/index/main/main-index.cc +++ b/icing/index/main/main-index.cc @@ -751,6 +751,13 @@ libtextclassifier3::StatusOr<DocumentId> MainIndex::TransferAndAddHits( old_pl_accessor.GetNextHitsBatch()); while (!tmp.empty()) { for (const Hit& hit : tmp) { + // A safety check to add robustness to the codebase, so to make sure that + // we never access invalid memory, in case that hit from the posting list + // is corrupted. + if (hit.document_id() < 0 || + hit.document_id() >= document_id_old_to_new.size()) { + continue; + } DocumentId new_document_id = document_id_old_to_new[hit.document_id()]; // Transfer the document id of the hit, if the document is not deleted // or outdated. diff --git a/icing/index/main/main-index_test.cc b/icing/index/main/main-index_test.cc index ac724b0..fa96e6c 100644 --- a/icing/index/main/main-index_test.cc +++ b/icing/index/main/main-index_test.cc @@ -38,6 +38,7 @@ namespace lib { namespace { using ::testing::ElementsAre; +using ::testing::Eq; using ::testing::IsEmpty; using ::testing::NiceMock; using ::testing::Return; @@ -90,7 +91,9 @@ class MainIndexTest : public testing::Test { std::string lite_index_file_name = index_dir_ + "/test_file.lite-idx.index"; LiteIndex::Options options(lite_index_file_name, - /*hit_buffer_want_merge_bytes=*/1024 * 1024); + /*hit_buffer_want_merge_bytes=*/1024 * 1024, + /*hit_buffer_sort_at_indexing=*/true, + /*hit_buffer_sort_threshold_bytes=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN(lite_index_, LiteIndex::Create(options, &icing_filesystem_)); @@ -361,7 +364,9 @@ TEST_F(MainIndexTest, MergeIndexToPreexisting) { // - Doc4 {"four", "foul" is_in_prefix_section=true} std::string lite_index_file_name2 = index_dir_ + "/test_file.lite-idx.index2"; LiteIndex::Options options(lite_index_file_name2, - /*hit_buffer_want_merge_bytes=*/1024 * 1024); + /*hit_buffer_want_merge_bytes=*/1024 * 1024, + /*hit_buffer_sort_at_indexing=*/true, + /*hit_buffer_sort_threshold_bytes=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN(lite_index_, LiteIndex::Create(options, &icing_filesystem_)); ICING_ASSERT_OK_AND_ASSIGN( @@ -531,30 +536,35 @@ TEST_F(MainIndexTest, PrefixNotRetrievedInExactSearch) { std::vector<SectionId>{doc1_hit.section_id()}))); } -TEST_F(MainIndexTest, SearchChainedPostingLists) { +TEST_F(MainIndexTest, + SearchChainedPostingListsShouldMergeSectionsAndTermFrequency) { // Index 2048 document with 3 hits in each document. When merged into the main // index, this will 1) lead to a chained posting list and 2) split at least // one document's hits across multiple posting lists. + const std::string term = "foot"; + ICING_ASSERT_OK_AND_ASSIGN( uint32_t tvi, - lite_index_->InsertTerm("foot", TermMatchType::EXACT_ONLY, kNamespace0)); + lite_index_->InsertTerm(term, TermMatchType::EXACT_ONLY, kNamespace0)); ICING_ASSERT_OK_AND_ASSIGN(uint32_t foot_term_id, term_id_codec_->EncodeTvi(tvi, TviType::LITE)); for (DocumentId document_id = 0; document_id < 2048; ++document_id) { - Hit doc_hit0(/*section_id=*/0, /*document_id=*/document_id, - Hit::kDefaultTermFrequency, - /*is_in_prefix_section=*/false); + Hit::TermFrequency term_frequency = static_cast<Hit::TermFrequency>( + document_id % Hit::kMaxTermFrequency + 1); + Hit doc_hit0( + /*section_id=*/0, /*document_id=*/document_id, term_frequency, + /*is_in_prefix_section=*/false); ICING_ASSERT_OK(lite_index_->AddHit(foot_term_id, doc_hit0)); - Hit doc_hit1(/*section_id=*/1, /*document_id=*/document_id, - Hit::kDefaultTermFrequency, - /*is_in_prefix_section=*/false); + Hit doc_hit1( + /*section_id=*/1, /*document_id=*/document_id, term_frequency, + /*is_in_prefix_section=*/false); ICING_ASSERT_OK(lite_index_->AddHit(foot_term_id, doc_hit1)); - Hit doc_hit2(/*section_id=*/2, /*document_id=*/document_id, - Hit::kDefaultTermFrequency, - /*is_in_prefix_section=*/false); + Hit doc_hit2( + /*section_id=*/2, /*document_id=*/document_id, term_frequency, + /*is_in_prefix_section=*/false); ICING_ASSERT_OK(lite_index_->AddHit(foot_term_id, doc_hit2)); } @@ -568,15 +578,35 @@ TEST_F(MainIndexTest, SearchChainedPostingLists) { // 3. Merge the lite index. ICING_ASSERT_OK(Merge(*lite_index_, *term_id_codec_, main_index.get())); // Get hits for all documents containing "foot" - which should be all of them. - std::vector<DocHitInfo> hits = - GetExactHits(main_index.get(), /*term_start_index=*/0, - /*unnormalized_term_length=*/0, "foot"); - EXPECT_THAT(hits, SizeIs(2048)); - EXPECT_THAT(hits.front(), - EqualsDocHitInfo(2047, std::vector<SectionId>{0, 1, 2})); - EXPECT_THAT(hits.back(), - EqualsDocHitInfo(0, std::vector<SectionId>{0, 1, 2})); + auto iterator = std::make_unique<DocHitInfoIteratorTermMainExact>( + main_index.get(), term, /*term_start_index=*/0, + /*unnormalized_term_length=*/0, kSectionIdMaskAll, + /*need_hit_term_frequency=*/true); + + DocumentId expected_document_id = 2047; + while (iterator->Advance().ok()) { + EXPECT_THAT(iterator->doc_hit_info(), + EqualsDocHitInfo(expected_document_id, + std::vector<SectionId>{0, 1, 2})); + + std::vector<TermMatchInfo> matched_terms_stats; + iterator->PopulateMatchedTermsStats(&matched_terms_stats); + + Hit::TermFrequency expected_term_frequency = + static_cast<Hit::TermFrequency>( + expected_document_id % Hit::kMaxTermFrequency + 1); + ASSERT_THAT(matched_terms_stats, SizeIs(1)); + EXPECT_THAT(matched_terms_stats[0].term, Eq(term)); + EXPECT_THAT(matched_terms_stats[0].term_frequencies[0], + Eq(expected_term_frequency)); + EXPECT_THAT(matched_terms_stats[0].term_frequencies[1], + Eq(expected_term_frequency)); + EXPECT_THAT(matched_terms_stats[0].term_frequencies[2], + Eq(expected_term_frequency)); + --expected_document_id; + } + EXPECT_THAT(expected_document_id, Eq(-1)); } TEST_F(MainIndexTest, MergeIndexBackfilling) { @@ -606,7 +636,9 @@ TEST_F(MainIndexTest, MergeIndexBackfilling) { // - Doc1 {"foot" is_in_prefix_section=false} std::string lite_index_file_name2 = index_dir_ + "/test_file.lite-idx.index2"; LiteIndex::Options options(lite_index_file_name2, - /*hit_buffer_want_merge_bytes=*/1024 * 1024); + /*hit_buffer_want_merge_bytes=*/1024 * 1024, + /*hit_buffer_sort_at_indexing=*/true, + /*hit_buffer_sort_threshold_bytes=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN(lite_index_, LiteIndex::Create(options, &icing_filesystem_)); ICING_ASSERT_OK_AND_ASSIGN( diff --git a/icing/index/numeric/dummy-numeric-index.h b/icing/index/numeric/dummy-numeric-index.h index 2c077a2..ce5fa45 100644 --- a/icing/index/numeric/dummy-numeric-index.h +++ b/icing/index/numeric/dummy-numeric-index.h @@ -183,29 +183,40 @@ class DummyNumericIndex : public NumericIndex<T> { std::string&& working_path) : NumericIndex<T>(filesystem, std::move(working_path), PersistentStorage::WorkingPathType::kDummy), - last_added_document_id_(kInvalidDocumentId) {} + dummy_crcs_buffer_( + std::make_unique<uint8_t[]>(sizeof(PersistentStorage::Crcs))), + last_added_document_id_(kInvalidDocumentId) { + memset(dummy_crcs_buffer_.get(), 0, sizeof(PersistentStorage::Crcs)); + } - libtextclassifier3::Status PersistStoragesToDisk() override { + libtextclassifier3::Status PersistStoragesToDisk(bool force) override { return libtextclassifier3::Status::OK; } - libtextclassifier3::Status PersistMetadataToDisk() override { + libtextclassifier3::Status PersistMetadataToDisk(bool force) override { return libtextclassifier3::Status::OK; } - libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum() override { + libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum(bool force) override { return Crc32(0); } - libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum() override { + libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum( + bool force) override { return Crc32(0); } - PersistentStorage::Crcs& crcs() override { return dummy_crcs_; } - const PersistentStorage::Crcs& crcs() const override { return dummy_crcs_; } + PersistentStorage::Crcs& crcs() override { + return *reinterpret_cast<PersistentStorage::Crcs*>( + dummy_crcs_buffer_.get()); + } + const PersistentStorage::Crcs& crcs() const override { + return *reinterpret_cast<const PersistentStorage::Crcs*>( + dummy_crcs_buffer_.get()); + } std::unordered_map<std::string, std::map<T, std::vector<BasicHit>>> storage_; - PersistentStorage::Crcs dummy_crcs_; + std::unique_ptr<uint8_t[]> dummy_crcs_buffer_; DocumentId last_added_document_id_; }; diff --git a/icing/index/numeric/integer-index-bucket-util.h b/icing/index/numeric/integer-index-bucket-util.h index 863bd01..d6fc245 100644 --- a/icing/index/numeric/integer-index-bucket-util.h +++ b/icing/index/numeric/integer-index-bucket-util.h @@ -61,7 +61,7 @@ struct DataRangeAndBucketInfo { // - Data slice (i.e. [start, end)) can be empty. // // REQUIRES: -// - original_key_lower <= original_key_upper +// - original_key_lower < original_key_upper // - num_data_threshold > 0 // - Keys of all data are in range [original_key_lower, original_key_upper] // diff --git a/icing/index/numeric/integer-index-storage.cc b/icing/index/numeric/integer-index-storage.cc index fa62b19..f0212da 100644 --- a/icing/index/numeric/integer-index-storage.cc +++ b/icing/index/numeric/integer-index-storage.cc @@ -45,6 +45,7 @@ #include "icing/index/numeric/posting-list-integer-index-serializer.h" #include "icing/schema/section.h" #include "icing/store/document-id.h" +#include "icing/util/crc32.h" #include "icing/util/status-macros.h" namespace icing { @@ -311,6 +312,11 @@ libtextclassifier3::Status IntegerIndexStorageIterator::Advance() { } bool IntegerIndexStorage::Options::IsValid() const { + if (num_data_threshold_for_bucket_split <= + kMinNumDataThresholdForBucketSplit) { + return false; + } + if (!HasCustomInitBuckets()) { return true; } @@ -403,12 +409,20 @@ libtextclassifier3::Status IntegerIndexStorage::AddKeys( return libtextclassifier3::Status::OK; } + SetDirty(); + std::sort(new_keys.begin(), new_keys.end()); // Dedupe auto last = std::unique(new_keys.begin(), new_keys.end()); new_keys.erase(last, new_keys.end()); + if (static_cast<int32_t>(new_keys.size()) > + std::numeric_limits<int32_t>::max() - info().num_data) { + return absl_ports::ResourceExhaustedError( + "# of keys in this integer index storage exceed the limit"); + } + // When adding keys into a bucket, we potentially split it into 2 new buckets // and one of them will be added into the unsorted bucket array. // When handling keys belonging to buckets in the unsorted bucket array, we @@ -649,6 +663,9 @@ libtextclassifier3::Status IntegerIndexStorage::TransferIndex( return lhs.get() < rhs.get(); }); + const int32_t num_data_threshold_for_bucket_merge = + kNumDataThresholdRatioForBucketMerge * + new_storage->options_.num_data_threshold_for_bucket_split; int64_t curr_key_lower = std::numeric_limits<int64_t>::min(); int64_t curr_key_upper = std::numeric_limits<int64_t>::min(); std::vector<IntegerIndexData> accumulated_data; @@ -687,7 +704,7 @@ libtextclassifier3::Status IntegerIndexStorage::TransferIndex( // - Flush accumulated_data and create a new bucket for them. // - OR merge new_data into accumulated_data and go to the next round. if (!accumulated_data.empty() && accumulated_data.size() + new_data.size() > - kNumDataThresholdForBucketMerge) { + num_data_threshold_for_bucket_merge) { // TODO(b/259743562): [Optimization 3] adjust upper bound to fit more data // from new_data to accumulated_data. ICING_RETURN_IF_ERROR(FlushDataIntoNewSortedBucket( @@ -879,9 +896,11 @@ IntegerIndexStorage::InitializeExistingFiles( IntegerIndexStorage::FlushDataIntoNewSortedBucket( int64_t key_lower, int64_t key_upper, std::vector<IntegerIndexData>&& data, IntegerIndexStorage* storage) { + storage->SetDirty(); + if (data.empty()) { - return storage->sorted_buckets_->Append( - Bucket(key_lower, key_upper, PostingListIdentifier::kInvalid)); + return storage->sorted_buckets_->Append(Bucket( + key_lower, key_upper, PostingListIdentifier::kInvalid, /*num_data=*/0)); } ICING_ASSIGN_OR_RETURN( @@ -891,10 +910,16 @@ IntegerIndexStorage::FlushDataIntoNewSortedBucket( data.end())); storage->info().num_data += data.size(); - return storage->sorted_buckets_->Append(Bucket(key_lower, key_upper, pl_id)); + return storage->sorted_buckets_->Append( + Bucket(key_lower, key_upper, pl_id, data.size())); } -libtextclassifier3::Status IntegerIndexStorage::PersistStoragesToDisk() { +libtextclassifier3::Status IntegerIndexStorage::PersistStoragesToDisk( + bool force) { + if (!force && !is_storage_dirty()) { + return libtextclassifier3::Status::OK; + } + ICING_RETURN_IF_ERROR(sorted_buckets_->PersistToDisk()); ICING_RETURN_IF_ERROR(unsorted_buckets_->PersistToDisk()); if (!flash_index_storage_->PersistToDisk()) { @@ -904,19 +929,35 @@ libtextclassifier3::Status IntegerIndexStorage::PersistStoragesToDisk() { return libtextclassifier3::Status::OK; } -libtextclassifier3::Status IntegerIndexStorage::PersistMetadataToDisk() { +libtextclassifier3::Status IntegerIndexStorage::PersistMetadataToDisk( + bool force) { + // We can skip persisting metadata to disk only if both info and storage are + // clean. + if (!force && !is_info_dirty() && !is_storage_dirty()) { + return libtextclassifier3::Status::OK; + } + // Changes should have been applied to the underlying file when using // MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, but call msync() as an // extra safety step to ensure they are written out. return metadata_mmapped_file_->PersistToDisk(); } -libtextclassifier3::StatusOr<Crc32> IntegerIndexStorage::ComputeInfoChecksum() { +libtextclassifier3::StatusOr<Crc32> IntegerIndexStorage::ComputeInfoChecksum( + bool force) { + if (!force && !is_info_dirty()) { + return Crc32(crcs().component_crcs.info_crc); + } + return info().ComputeChecksum(); } libtextclassifier3::StatusOr<Crc32> -IntegerIndexStorage::ComputeStoragesChecksum() { +IntegerIndexStorage::ComputeStoragesChecksum(bool force) { + if (!force && !is_storage_dirty()) { + return Crc32(crcs().component_crcs.storages_crc); + } + // Compute crcs ICING_ASSIGN_OR_RETURN(Crc32 sorted_buckets_crc, sorted_buckets_->ComputeChecksum()); @@ -933,6 +974,89 @@ IntegerIndexStorage::AddKeysIntoBucketAndSplitIfNecessary( const std::vector<int64_t>::const_iterator& it_start, const std::vector<int64_t>::const_iterator& it_end, FileBackedVector<Bucket>::MutableView& mutable_bucket) { + int32_t num_data_in_bucket = mutable_bucket.Get().num_data(); + int32_t num_new_data = std::distance(it_start, it_end); + if (mutable_bucket.Get().key_lower() < mutable_bucket.Get().key_upper() && + num_new_data + num_data_in_bucket > + options_.num_data_threshold_for_bucket_split) { + // Split bucket. + + // 1. Read all data and free all posting lists. + std::vector<IntegerIndexData> all_data; + if (mutable_bucket.Get().posting_list_identifier().is_valid()) { + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<PostingListIntegerIndexAccessor> pl_accessor, + PostingListIntegerIndexAccessor::CreateFromExisting( + flash_index_storage_.get(), posting_list_serializer_, + mutable_bucket.Get().posting_list_identifier())); + ICING_ASSIGN_OR_RETURN(all_data, pl_accessor->GetAllDataAndFree()); + } + + // 2. Append all new data. + all_data.reserve(all_data.size() + num_new_data); + for (auto it = it_start; it != it_end; ++it) { + all_data.push_back(IntegerIndexData(section_id, document_id, *it)); + } + + // 3. Run bucket splitting algorithm to decide new buckets and dispatch + // data. + // - # of data in a full bucket = + // options_.num_data_threshold_for_bucket_split. + // - Bucket splitting logic will be invoked if adding new data + // (num_new_data >= 1) into a full bucket. + // - In order to achieve good (amortized) time complexity, we want # of + // data in new buckets to be around half_of_threshold (i.e. + // options_.num_data_threshold_for_bucket_split / 2). + // - Using half_of_threshold as the cutoff threshold will cause splitting + // buckets with [half_of_threshold, half_of_threshold, num_new_data] + // data, which is not ideal because num_new_data is usually small. + // - Thus, we pick (half_of_threshold + kNumDataAfterSplitAdjustment) as + // the cutoff threshold to avoid over-splitting. It can tolerate + // num_new_data up to (2 * kNumDataAfterSplitAdjustment) and + // split only 2 buckets (instead of 3) with + // [half_of_threshold + kNumDataAfterSplitAdjustment, + // half_of_threshold + (kNumDataAfterSplitAdjustment - num_new_data)]. + int32_t cutoff_threshold = + options_.num_data_threshold_for_bucket_split / 2 + + kNumDataAfterSplitAdjustment; + std::vector<integer_index_bucket_util::DataRangeAndBucketInfo> + new_bucket_infos = integer_index_bucket_util::Split( + all_data, mutable_bucket.Get().key_lower(), + mutable_bucket.Get().key_upper(), cutoff_threshold); + if (new_bucket_infos.empty()) { + ICING_LOG(WARNING) + << "No buckets after splitting. This should not happen."; + return absl_ports::InternalError("Split error"); + } + + // 4. Flush data and create new buckets. + std::vector<Bucket> new_buckets; + for (int i = 0; i < new_bucket_infos.size(); ++i) { + int32_t num_data_in_new_bucket = + std::distance(new_bucket_infos[i].start, new_bucket_infos[i].end); + ICING_ASSIGN_OR_RETURN( + PostingListIdentifier pl_id, + FlushDataIntoPostingLists( + flash_index_storage_.get(), posting_list_serializer_, + new_bucket_infos[i].start, new_bucket_infos[i].end)); + if (i == 0) { + // Reuse mutable_bucket + mutable_bucket.Get().set_key_lower(new_bucket_infos[i].key_lower); + mutable_bucket.Get().set_key_upper(new_bucket_infos[i].key_upper); + mutable_bucket.Get().set_posting_list_identifier(pl_id); + mutable_bucket.Get().set_num_data(num_data_in_new_bucket); + } else { + new_buckets.push_back(Bucket(new_bucket_infos[i].key_lower, + new_bucket_infos[i].key_upper, pl_id, + num_data_in_new_bucket)); + } + } + + return new_buckets; + } + + // Otherwise, we don't need to split bucket. Just simply add all new data into + // the bucket. std::unique_ptr<PostingListIntegerIndexAccessor> pl_accessor; if (mutable_bucket.Get().posting_list_identifier().is_valid()) { ICING_ASSIGN_OR_RETURN( @@ -946,68 +1070,6 @@ IntegerIndexStorage::AddKeysIntoBucketAndSplitIfNecessary( } for (auto it = it_start; it != it_end; ++it) { - if (mutable_bucket.Get().key_lower() < mutable_bucket.Get().key_upper() && - pl_accessor->WantsSplit()) { - // If the bucket needs split (max size and full) and is splittable, then - // we perform bucket splitting. - - // 1. Finalize the current posting list accessor. - PostingListAccessor::FinalizeResult result = - std::move(*pl_accessor).Finalize(); - if (!result.status.ok()) { - return result.status; - } - - // 2. Create another posting list accessor instance. Read all data and - // free all posting lists. - ICING_ASSIGN_OR_RETURN( - pl_accessor, - PostingListIntegerIndexAccessor::CreateFromExisting( - flash_index_storage_.get(), posting_list_serializer_, result.id)); - ICING_ASSIGN_OR_RETURN(std::vector<IntegerIndexData> all_data, - pl_accessor->GetAllDataAndFree()); - - // 3. Append all remaining new data. - all_data.reserve(all_data.size() + std::distance(it, it_end)); - for (; it != it_end; ++it) { - all_data.push_back(IntegerIndexData(section_id, document_id, *it)); - } - - // 4. Run bucket splitting algorithm to decide new buckets and dispatch - // data. - std::vector<integer_index_bucket_util::DataRangeAndBucketInfo> - new_bucket_infos = integer_index_bucket_util::Split( - all_data, mutable_bucket.Get().key_lower(), - mutable_bucket.Get().key_upper(), - kNumDataThresholdForBucketSplit); - if (new_bucket_infos.empty()) { - ICING_LOG(WARNING) - << "No buckets after splitting. This should not happen."; - return absl_ports::InternalError("Split error"); - } - - // 5. Flush data. - std::vector<Bucket> new_buckets; - for (int i = 0; i < new_bucket_infos.size(); ++i) { - ICING_ASSIGN_OR_RETURN( - PostingListIdentifier pl_id, - FlushDataIntoPostingLists( - flash_index_storage_.get(), posting_list_serializer_, - new_bucket_infos[i].start, new_bucket_infos[i].end)); - if (i == 0) { - // Reuse mutable_bucket - mutable_bucket.Get().set_key_lower(new_bucket_infos[i].key_lower); - mutable_bucket.Get().set_key_upper(new_bucket_infos[i].key_upper); - mutable_bucket.Get().set_posting_list_identifier(pl_id); - } else { - new_buckets.push_back(Bucket(new_bucket_infos[i].key_lower, - new_bucket_infos[i].key_upper, pl_id)); - } - } - - return new_buckets; - } - ICING_RETURN_IF_ERROR(pl_accessor->PrependData( IntegerIndexData(section_id, document_id, *it))); } @@ -1022,6 +1084,9 @@ IntegerIndexStorage::AddKeysIntoBucketAndSplitIfNecessary( } mutable_bucket.Get().set_posting_list_identifier(result.id); + // We've already verified num_new_data won't exceed the limit of the entire + // storage, so it is safe to add to the counter of the bucket. + mutable_bucket.Get().set_num_data(num_data_in_bucket + num_new_data); return std::vector<Bucket>(); } diff --git a/icing/index/numeric/integer-index-storage.h b/icing/index/numeric/integer-index-storage.h index 9f2e58c..0c1afbb 100644 --- a/icing/index/numeric/integer-index-storage.h +++ b/icing/index/numeric/integer-index-storage.h @@ -75,7 +75,7 @@ namespace lib { class IntegerIndexStorage : public PersistentStorage { public: struct Info { - static constexpr int32_t kMagic = 0xc4bf0ccc; + static constexpr int32_t kMagic = 0x6470e547; int32_t magic; int32_t num_data; @@ -99,10 +99,12 @@ class IntegerIndexStorage : public PersistentStorage { explicit Bucket(int64_t key_lower, int64_t key_upper, PostingListIdentifier posting_list_identifier = - PostingListIdentifier::kInvalid) + PostingListIdentifier::kInvalid, + int32_t num_data = 0) : key_lower_(key_lower), key_upper_(key_upper), - posting_list_identifier_(posting_list_identifier) {} + posting_list_identifier_(posting_list_identifier), + num_data_(num_data) {} bool operator<(const Bucket& other) const { return key_lower_ < other.key_lower_; @@ -130,12 +132,16 @@ class IntegerIndexStorage : public PersistentStorage { posting_list_identifier_ = posting_list_identifier; } + int32_t num_data() const { return num_data_; } + void set_num_data(int32_t num_data) { num_data_ = num_data; } + private: int64_t key_lower_; int64_t key_upper_; PostingListIdentifier posting_list_identifier_; + int32_t num_data_; } __attribute__((packed)); - static_assert(sizeof(Bucket) == 20, ""); + static_assert(sizeof(Bucket) == 24, ""); static_assert(sizeof(Bucket) == FileBackedVector<Bucket>::kElementTypeSize, "Bucket type size is inconsistent with FileBackedVector " "element type size"); @@ -146,15 +152,31 @@ class IntegerIndexStorage : public PersistentStorage { "Max # of buckets cannot fit into FileBackedVector"); struct Options { - explicit Options(bool pre_mapping_fbv_in) - : pre_mapping_fbv(pre_mapping_fbv_in) {} + // - According to the benchmark result, the more # of buckets, the higher + // latency for range query. Therefore, this number cannot be too small to + // avoid splitting bucket too aggressively. + // - We use `num_data_threshold_for_bucket_split / 2 + 5` as the cutoff + // threshold after splitting. This number cannot be too small (e.g. 10) + // because in this case we will have similar # of data in a single bucket + // before and after splitting, which contradicts the purpose of splitting. + // - For convenience, let's set 64 as the minimum value. + static constexpr int32_t kMinNumDataThresholdForBucketSplit = 64; + + explicit Options(int32_t num_data_threshold_for_bucket_split_in, + bool pre_mapping_fbv_in) + : num_data_threshold_for_bucket_split( + num_data_threshold_for_bucket_split_in), + pre_mapping_fbv(pre_mapping_fbv_in) {} explicit Options(std::vector<Bucket> custom_init_sorted_buckets_in, std::vector<Bucket> custom_init_unsorted_buckets_in, + int32_t num_data_threshold_for_bucket_split_in, bool pre_mapping_fbv_in) : custom_init_sorted_buckets(std::move(custom_init_sorted_buckets_in)), custom_init_unsorted_buckets( std::move(custom_init_unsorted_buckets_in)), + num_data_threshold_for_bucket_split( + num_data_threshold_for_bucket_split_in), pre_mapping_fbv(pre_mapping_fbv_in) {} bool IsValid() const; @@ -172,6 +194,14 @@ class IntegerIndexStorage : public PersistentStorage { std::vector<Bucket> custom_init_sorted_buckets; std::vector<Bucket> custom_init_unsorted_buckets; + // Threshold for invoking bucket splitting. If # of data in a bucket exceeds + // this number after adding new data, then it will invoke bucket splitting + // logic. + // + // Note: num_data_threshold_for_bucket_split should be >= + // kMinNumDataThresholdForBucketSplit. + int32_t num_data_threshold_for_bucket_split; + // Flag indicating whether memory map max possible file size for underlying // FileBackedVector before growing the actual file size. bool pre_mapping_fbv; @@ -188,28 +218,25 @@ class IntegerIndexStorage : public PersistentStorage { WorkingPathType::kDirectory; static constexpr std::string_view kFilePrefix = "integer_index_storage"; - // # of data threshold for bucket merging during optimization (TransferIndex). - // If total # data of adjacent buckets exceed this value, then flush the - // accumulated data. Otherwise merge buckets and their data. - // - // Calculated by: 0.7 * (kMaxPostingListSize / sizeof(IntegerIndexData)), - // where kMaxPostingListSize = (kPageSize - sizeof(IndexBlock::BlockHeader)). - static constexpr int32_t kNumDataThresholdForBucketMerge = 240; - - // # of data threshold for bucket splitting during indexing (AddKeys). - // When the posting list of a bucket is full, we will try to split data into - // multiple buckets according to their keys. In order to achieve good - // (amortized) time complexity, we want # of data in new buckets to be at most - // half # of elements in a full posting list. + // Default # of data threshold for bucket splitting during indexing (AddKeys). + // When # of data in a bucket reaches this number, we will try to split data + // into multiple buckets according to their keys. + static constexpr int32_t kDefaultNumDataThresholdForBucketSplit = 65536; + + // # of data threshold for bucket merging during optimization (TransferIndex) + // = kNumDataThresholdRatioForBucketMerge * + // options.num_data_threshold_for_bucket_split // - // Calculated by: 0.5 * (kMaxPostingListSize / sizeof(IntegerIndexData)), - // where kMaxPostingListSize = (kPageSize - sizeof(IndexBlock::BlockHeader)). - static constexpr int32_t kNumDataThresholdForBucketSplit = 170; + // If total # data of adjacent buckets exceed this threshold, then flush the + // accumulated data. Otherwise merge buckets and their data. + static constexpr double kNumDataThresholdRatioForBucketMerge = 0.7; // Length threshold to sort and merge unsorted buckets into sorted buckets. If // the length of unsorted_buckets exceed the threshold, then call // SortBuckets(). - static constexpr int32_t kUnsortedBucketsLengthThreshold = 50; + // TODO(b/259743562): decide if removing unsorted buckets given that we + // changed bucket splitting threshold and # of buckets are small now. + static constexpr int32_t kUnsortedBucketsLengthThreshold = 5; // Creates a new IntegerIndexStorage instance to index integers (for a single // property). If any of the underlying file is missing, then delete the whole @@ -272,6 +299,8 @@ class IntegerIndexStorage : public PersistentStorage { // // Returns: // - OK on success + // - RESOURCE_EXHAUSTED_ERROR if # of integers in this storage exceed + // INT_MAX after adding new_keys // - Any FileBackedVector or PostingList errors libtextclassifier3::Status AddKeys(DocumentId document_id, SectionId section_id, @@ -314,6 +343,8 @@ class IntegerIndexStorage : public PersistentStorage { int32_t num_data() const { return info().num_data; } private: + static constexpr int8_t kNumDataAfterSplitAdjustment = 5; + explicit IntegerIndexStorage( const Filesystem& filesystem, std::string&& working_path, Options&& options, @@ -329,7 +360,9 @@ class IntegerIndexStorage : public PersistentStorage { metadata_mmapped_file_(std::move(metadata_mmapped_file)), sorted_buckets_(std::move(sorted_buckets)), unsorted_buckets_(std::move(unsorted_buckets)), - flash_index_storage_(std::move(flash_index_storage)) {} + flash_index_storage_(std::move(flash_index_storage)), + is_info_dirty_(false), + is_storage_dirty_(false) {} static libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndexStorage>> InitializeNewFiles( @@ -360,20 +393,20 @@ class IntegerIndexStorage : public PersistentStorage { // Returns: // - OK on success // - INTERNAL_ERROR on I/O error - libtextclassifier3::Status PersistStoragesToDisk() override; + libtextclassifier3::Status PersistStoragesToDisk(bool force) override; // Flushes contents of metadata file. // // Returns: // - OK on success // - INTERNAL_ERROR on I/O error - libtextclassifier3::Status PersistMetadataToDisk() override; + libtextclassifier3::Status PersistMetadataToDisk(bool force) override; // Computes and returns Info checksum. // // Returns: // - Crc of the Info on success - libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum() override; + libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum(bool force) override; // Computes and returns all storages checksum. Checksums of sorted_buckets_, // unsorted_buckets_ will be combined together by XOR. @@ -382,7 +415,8 @@ class IntegerIndexStorage : public PersistentStorage { // Returns: // - Crc of all storages on success // - INTERNAL_ERROR if any data inconsistency - libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum() override; + libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum( + bool force) override; // Helper function to add keys in range [it_start, it_end) into the given // bucket. It handles the bucket and its corresponding posting list(s) to make @@ -442,6 +476,17 @@ class IntegerIndexStorage : public PersistentStorage { kInfoMetadataFileOffset); } + void SetInfoDirty() { is_info_dirty_ = true; } + // When storage is dirty, we have to set info dirty as well. So just expose + // SetDirty to set both. + void SetDirty() { + is_info_dirty_ = true; + is_storage_dirty_ = true; + } + + bool is_info_dirty() const { return is_info_dirty_; } + bool is_storage_dirty() const { return is_storage_dirty_; } + Options options_; PostingListIntegerIndexSerializer* posting_list_serializer_; // Does not own. @@ -450,6 +495,9 @@ class IntegerIndexStorage : public PersistentStorage { std::unique_ptr<FileBackedVector<Bucket>> sorted_buckets_; std::unique_ptr<FileBackedVector<Bucket>> unsorted_buckets_; std::unique_ptr<FlashIndexStorage> flash_index_storage_; + + bool is_info_dirty_; + bool is_storage_dirty_; }; } // namespace lib diff --git a/icing/index/numeric/integer-index-storage_benchmark.cc b/icing/index/numeric/integer-index-storage_benchmark.cc index bf5f134..85d381d 100644 --- a/icing/index/numeric/integer-index-storage_benchmark.cc +++ b/icing/index/numeric/integer-index-storage_benchmark.cc @@ -68,6 +68,8 @@ using ::testing::Eq; using ::testing::IsEmpty; using ::testing::SizeIs; +static constexpr int32_t kNumDataThresholdForBucketSplit = + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit; static constexpr bool kPreMappingFbv = true; static constexpr SectionId kDefaultSectionId = 12; @@ -150,11 +152,13 @@ void BM_Index(benchmark::State& state) { state.PauseTiming(); benchmark.filesystem.DeleteDirectoryRecursively( benchmark.working_path.c_str()); - ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create( - benchmark.filesystem, benchmark.working_path, - IntegerIndexStorage::Options(kPreMappingFbv), - &benchmark.posting_list_serializer)); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<IntegerIndexStorage> storage, + IntegerIndexStorage::Create( + benchmark.filesystem, benchmark.working_path, + IntegerIndexStorage::Options(kNumDataThresholdForBucketSplit, + kPreMappingFbv), + &benchmark.posting_list_serializer)); state.ResumeTiming(); for (int i = 0; i < num_keys; ++i) { @@ -210,11 +214,13 @@ void BM_BatchIndex(benchmark::State& state) { state.PauseTiming(); benchmark.filesystem.DeleteDirectoryRecursively( benchmark.working_path.c_str()); - ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create( - benchmark.filesystem, benchmark.working_path, - IntegerIndexStorage::Options(kPreMappingFbv), - &benchmark.posting_list_serializer)); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<IntegerIndexStorage> storage, + IntegerIndexStorage::Create( + benchmark.filesystem, benchmark.working_path, + IntegerIndexStorage::Options(kNumDataThresholdForBucketSplit, + kPreMappingFbv), + &benchmark.posting_list_serializer)); std::vector<int64_t> keys_copy(keys); state.ResumeTiming(); @@ -263,9 +269,11 @@ void BM_ExactQuery(benchmark::State& state) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(benchmark.filesystem, benchmark.working_path, - IntegerIndexStorage::Options(kPreMappingFbv), - &benchmark.posting_list_serializer)); + IntegerIndexStorage::Create( + benchmark.filesystem, benchmark.working_path, + IntegerIndexStorage::Options(kNumDataThresholdForBucketSplit, + kPreMappingFbv), + &benchmark.posting_list_serializer)); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<NumberGenerator<int64_t>> generator, CreateIntegerGenerator(distribution_type, kDefaultSeed, num_keys)); @@ -340,9 +348,11 @@ void BM_RangeQueryAll(benchmark::State& state) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(benchmark.filesystem, benchmark.working_path, - IntegerIndexStorage::Options(kPreMappingFbv), - &benchmark.posting_list_serializer)); + IntegerIndexStorage::Create( + benchmark.filesystem, benchmark.working_path, + IntegerIndexStorage::Options(kNumDataThresholdForBucketSplit, + kPreMappingFbv), + &benchmark.posting_list_serializer)); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<NumberGenerator<int64_t>> generator, CreateIntegerGenerator(distribution_type, kDefaultSeed, num_keys)); diff --git a/icing/index/numeric/integer-index-storage_test.cc b/icing/index/numeric/integer-index-storage_test.cc index 4d4e665..8675172 100644 --- a/icing/index/numeric/integer-index-storage_test.cc +++ b/icing/index/numeric/integer-index-storage_test.cc @@ -30,8 +30,6 @@ #include "icing/file/file-backed-vector.h" #include "icing/file/filesystem.h" #include "icing/file/persistent-storage.h" -#include "icing/file/posting_list/flash-index-storage.h" -#include "icing/file/posting_list/index-block.h" #include "icing/file/posting_list/posting-list-identifier.h" #include "icing/index/hit/doc-hit-info.h" #include "icing/index/iterator/doc-hit-info-iterator.h" @@ -106,7 +104,32 @@ libtextclassifier3::StatusOr<std::vector<DocHitInfo>> Query( } TEST_P(IntegerIndexStorageTest, OptionsEmptyCustomInitBucketsShouldBeValid) { - EXPECT_THAT(Options(/*pre_mapping_fbv_in=*/GetParam()).IsValid(), IsTrue()); + EXPECT_THAT( + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsTrue()); +} + +TEST_P(IntegerIndexStorageTest, OptionsInvalidNumDataThresholdForBucketSplit) { + EXPECT_THAT(Options(/*custom_init_sorted_buckets_in=*/{}, + /*custom_init_unsorted_buckets_in=*/{}, + /*num_data_threshold_for_bucket_split=*/-1, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsFalse()); + EXPECT_THAT(Options(/*custom_init_sorted_buckets_in=*/{}, + /*custom_init_unsorted_buckets_in=*/{}, + /*num_data_threshold_for_bucket_split=*/0, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsFalse()); + EXPECT_THAT(Options(/*custom_init_sorted_buckets_in=*/{}, + /*custom_init_unsorted_buckets_in=*/{}, + /*num_data_threshold_for_bucket_split=*/63, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsFalse()); } TEST_P(IntegerIndexStorageTest, OptionsInvalidCustomInitBucketsRange) { @@ -116,6 +139,7 @@ TEST_P(IntegerIndexStorageTest, OptionsInvalidCustomInitBucketsRange) { {Bucket(std::numeric_limits<int64_t>::min(), 5), Bucket(9, 6)}, /*custom_init_unsorted_buckets_in=*/ {Bucket(10, std::numeric_limits<int64_t>::max())}, + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()) .IsValid(), IsFalse()); @@ -126,6 +150,7 @@ TEST_P(IntegerIndexStorageTest, OptionsInvalidCustomInitBucketsRange) { {Bucket(10, std::numeric_limits<int64_t>::max())}, /*custom_init_unsorted_buckets_in=*/ {Bucket(std::numeric_limits<int64_t>::min(), 5), Bucket(9, 6)}, + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()) .IsValid(), IsFalse()); @@ -138,91 +163,109 @@ TEST_P(IntegerIndexStorageTest, ASSERT_THAT(valid_posting_list_identifier.is_valid(), IsTrue()); // Invalid custom init sorted bucket - EXPECT_THAT(Options(/*custom_init_sorted_buckets_in=*/ - {Bucket(std::numeric_limits<int64_t>::min(), - std::numeric_limits<int64_t>::max(), - valid_posting_list_identifier)}, - /*custom_init_unsorted_buckets_in=*/{}, - /*pre_mapping_fbv_in=*/GetParam()) - .IsValid(), - IsFalse()); + EXPECT_THAT( + Options(/*custom_init_sorted_buckets_in=*/ + {Bucket(std::numeric_limits<int64_t>::min(), + std::numeric_limits<int64_t>::max(), + valid_posting_list_identifier)}, + /*custom_init_unsorted_buckets_in=*/{}, + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsFalse()); // Invalid custom init unsorted bucket - EXPECT_THAT(Options(/*custom_init_sorted_buckets_in=*/{}, - /*custom_init_unsorted_buckets_in=*/ - {Bucket(std::numeric_limits<int64_t>::min(), - std::numeric_limits<int64_t>::max(), - valid_posting_list_identifier)}, - /*pre_mapping_fbv_in=*/GetParam()) - .IsValid(), - IsFalse()); + EXPECT_THAT( + Options(/*custom_init_sorted_buckets_in=*/{}, + /*custom_init_unsorted_buckets_in=*/ + {Bucket(std::numeric_limits<int64_t>::min(), + std::numeric_limits<int64_t>::max(), + valid_posting_list_identifier)}, + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsFalse()); } TEST_P(IntegerIndexStorageTest, OptionsInvalidCustomInitBucketsOverlapping) { // sorted buckets overlap - EXPECT_THAT(Options(/*custom_init_sorted_buckets_in=*/ - {Bucket(std::numeric_limits<int64_t>::min(), -100), - Bucket(-100, std::numeric_limits<int64_t>::max())}, - /*custom_init_unsorted_buckets_in=*/{}, - /*pre_mapping_fbv_in=*/GetParam()) - .IsValid(), - IsFalse()); + EXPECT_THAT( + Options(/*custom_init_sorted_buckets_in=*/ + {Bucket(std::numeric_limits<int64_t>::min(), -100), + Bucket(-100, std::numeric_limits<int64_t>::max())}, + /*custom_init_unsorted_buckets_in=*/{}, + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsFalse()); // unsorted buckets overlap - EXPECT_THAT(Options(/*custom_init_sorted_buckets_in=*/{}, - /*custom_init_unsorted_buckets_in=*/ - {Bucket(-100, std::numeric_limits<int64_t>::max()), - Bucket(std::numeric_limits<int64_t>::min(), -100)}, - /*pre_mapping_fbv_in=*/GetParam()) - .IsValid(), - IsFalse()); + EXPECT_THAT( + Options(/*custom_init_sorted_buckets_in=*/{}, + /*custom_init_unsorted_buckets_in=*/ + {Bucket(-100, std::numeric_limits<int64_t>::max()), + Bucket(std::numeric_limits<int64_t>::min(), -100)}, + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsFalse()); // Cross buckets overlap - EXPECT_THAT(Options(/*custom_init_sorted_buckets_in=*/ - {Bucket(std::numeric_limits<int64_t>::min(), -100), - Bucket(-99, 0)}, - /*custom_init_unsorted_buckets_in=*/ - {Bucket(200, std::numeric_limits<int64_t>::max()), - Bucket(0, 50), Bucket(51, 199)}, - /*pre_mapping_fbv_in=*/GetParam()) - .IsValid(), - IsFalse()); + EXPECT_THAT( + Options(/*custom_init_sorted_buckets_in=*/ + {Bucket(std::numeric_limits<int64_t>::min(), -100), + Bucket(-99, 0)}, + /*custom_init_unsorted_buckets_in=*/ + {Bucket(200, std::numeric_limits<int64_t>::max()), Bucket(0, 50), + Bucket(51, 199)}, + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsFalse()); } TEST_P(IntegerIndexStorageTest, OptionsInvalidCustomInitBucketsUnion) { // Missing INT64_MAX - EXPECT_THAT(Options(/*custom_init_sorted_buckets_in=*/ - {Bucket(std::numeric_limits<int64_t>::min(), -100), - Bucket(-99, 0)}, - /*custom_init_unsorted_buckets_in=*/ - {Bucket(1, 1000)}, /*pre_mapping_fbv_in=*/GetParam()) - .IsValid(), - IsFalse()); + EXPECT_THAT( + Options(/*custom_init_sorted_buckets_in=*/ + {Bucket(std::numeric_limits<int64_t>::min(), -100), + Bucket(-99, 0)}, + /*custom_init_unsorted_buckets_in=*/{Bucket(1, 1000)}, + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsFalse()); // Missing INT64_MIN - EXPECT_THAT(Options(/*custom_init_sorted_buckets_in=*/ - {Bucket(-200, -100), Bucket(-99, 0)}, - /*custom_init_unsorted_buckets_in=*/ - {Bucket(1, std::numeric_limits<int64_t>::max())}, - /*pre_mapping_fbv_in=*/GetParam()) - .IsValid(), - IsFalse()); + EXPECT_THAT( + Options(/*custom_init_sorted_buckets_in=*/ + {Bucket(-200, -100), Bucket(-99, 0)}, + /*custom_init_unsorted_buckets_in=*/ + {Bucket(1, std::numeric_limits<int64_t>::max())}, + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsFalse()); // Missing some intermediate ranges - EXPECT_THAT(Options(/*custom_init_sorted_buckets_in=*/ - {Bucket(std::numeric_limits<int64_t>::min(), -100)}, - /*custom_init_unsorted_buckets_in=*/ - {Bucket(1, std::numeric_limits<int64_t>::max())}, - /*pre_mapping_fbv_in=*/GetParam()) - .IsValid(), - IsFalse()); + EXPECT_THAT( + Options(/*custom_init_sorted_buckets_in=*/ + {Bucket(std::numeric_limits<int64_t>::min(), -100)}, + /*custom_init_unsorted_buckets_in=*/ + {Bucket(1, std::numeric_limits<int64_t>::max())}, + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()) + .IsValid(), + IsFalse()); } TEST_P(IntegerIndexStorageTest, InvalidWorkingPath) { EXPECT_THAT( IntegerIndexStorage::Create( filesystem_, "/dev/null/integer_index_storage_test", - Options(/*pre_mapping_fbv_in=*/GetParam()), serializer_.get()), + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get()), StatusIs(libtextclassifier3::StatusCode::INTERNAL)); } @@ -232,6 +275,7 @@ TEST_P(IntegerIndexStorageTest, CreateWithInvalidOptionsShouldFail) { /*custom_init_unsorted_buckets_in=*/ {Bucket(-100, std::numeric_limits<int64_t>::max()), Bucket(std::numeric_limits<int64_t>::min(), -100)}, + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()); ASSERT_THAT(invalid_options.IsValid(), IsFalse()); @@ -246,9 +290,11 @@ TEST_P(IntegerIndexStorageTest, InitializeNewFiles) { ASSERT_FALSE(filesystem_.DirectoryExists(working_path_.c_str())); ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); ICING_ASSERT_OK(storage->PersistToDisk()); } @@ -290,9 +336,11 @@ TEST_P(IntegerIndexStorageTest, // Create new integer index storage ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); // Insert some data. ICING_ASSERT_OK(storage->AddKeys(/*document_id=*/0, /*section_id=*/20, @@ -305,9 +353,11 @@ TEST_P(IntegerIndexStorageTest, // Without calling PersistToDisk, checksums will not be recomputed or synced // to disk, so initializing another instance on the same files should fail. EXPECT_THAT( - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get()), + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } @@ -315,9 +365,11 @@ TEST_P(IntegerIndexStorageTest, InitializationShouldSucceedWithPersistToDisk) { // Create new integer index storage ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage1, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); // Insert some data. ICING_ASSERT_OK(storage1->AddKeys(/*document_id=*/0, /*section_id=*/20, @@ -339,9 +391,11 @@ TEST_P(IntegerIndexStorageTest, InitializationShouldSucceedWithPersistToDisk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage2, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); EXPECT_THAT( Query(storage2.get(), /*key_lower=*/std::numeric_limits<int64_t>::min(), /*key_upper=*/std::numeric_limits<int64_t>::max()), @@ -355,9 +409,11 @@ TEST_P(IntegerIndexStorageTest, InitializationShouldSucceedAfterDestruction) { // Create new integer index storage ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); // Insert some data. ICING_ASSERT_OK(storage->AddKeys(/*document_id=*/0, /*section_id=*/20, @@ -380,9 +436,11 @@ TEST_P(IntegerIndexStorageTest, InitializationShouldSucceedAfterDestruction) { // we should be able to get the same contents. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); EXPECT_THAT( Query(storage.get(), /*key_lower=*/std::numeric_limits<int64_t>::min(), /*key_upper=*/std::numeric_limits<int64_t>::max()), @@ -397,9 +455,11 @@ TEST_P(IntegerIndexStorageTest, // Create new integer index storage ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); ICING_ASSERT_OK(storage->AddKeys(kDefaultDocumentId, kDefaultSectionId, /*new_keys=*/{0, 100, -100})); @@ -428,7 +488,9 @@ TEST_P(IntegerIndexStorageTest, libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndexStorage>> storage_or = IntegerIndexStorage::Create( filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), serializer_.get()); + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get()); EXPECT_THAT(storage_or, StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT(storage_or.status().error_message(), @@ -442,9 +504,11 @@ TEST_P(IntegerIndexStorageTest, // Create new integer index storage ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); ICING_ASSERT_OK(storage->AddKeys(kDefaultDocumentId, kDefaultSectionId, /*new_keys=*/{0, 100, -100})); @@ -474,7 +538,9 @@ TEST_P(IntegerIndexStorageTest, libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndexStorage>> storage_or = IntegerIndexStorage::Create( filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), serializer_.get()); + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get()); EXPECT_THAT(storage_or, StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT(storage_or.status().error_message(), @@ -488,9 +554,11 @@ TEST_P(IntegerIndexStorageTest, // Create new integer index storage ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); ICING_ASSERT_OK(storage->AddKeys(kDefaultDocumentId, kDefaultSectionId, /*new_keys=*/{0, 100, -100})); @@ -522,7 +590,9 @@ TEST_P(IntegerIndexStorageTest, libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndexStorage>> storage_or = IntegerIndexStorage::Create( filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), serializer_.get()); + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get()); EXPECT_THAT(storage_or, StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT(storage_or.status().error_message(), @@ -536,9 +606,11 @@ TEST_P(IntegerIndexStorageTest, // Create new integer index storage ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); ICING_ASSERT_OK(storage->AddKeys(kDefaultDocumentId, kDefaultSectionId, /*new_keys=*/{0, 100, -100})); @@ -572,7 +644,9 @@ TEST_P(IntegerIndexStorageTest, libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndexStorage>> storage_or = IntegerIndexStorage::Create( filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), serializer_.get()); + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get()); EXPECT_THAT(storage_or, StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT(storage_or.status().error_message(), @@ -586,14 +660,119 @@ TEST_P(IntegerIndexStorageTest, InvalidQuery) { // Create new integer index storage ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); EXPECT_THAT( storage->GetIterator(/*query_key_lower=*/0, /*query_key_upper=*/-1), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } +TEST_P(IntegerIndexStorageTest, AddKeysShouldUpdateNumData) { + // We use predefined custom buckets to initialize new integer index storage + // and create some test keys accordingly. + std::vector<Bucket> custom_init_sorted_buckets = { + Bucket(-1000, -100), Bucket(0, 100), Bucket(150, 199), Bucket(200, 300), + Bucket(301, 999)}; + std::vector<Bucket> custom_init_unsorted_buckets = { + Bucket(1000, std::numeric_limits<int64_t>::max()), Bucket(-99, -1), + Bucket(101, 149), Bucket(std::numeric_limits<int64_t>::min(), -1001)}; + { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<IntegerIndexStorage> storage, + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(std::move(custom_init_sorted_buckets), + std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); + + // Add some keys into buckets [(-1000,-100), (200,300), (-99,-1)]. + EXPECT_THAT(storage->AddKeys(/*document_id=*/0, kDefaultSectionId, + /*new_keys=*/{-51, -500}), + IsOk()); + EXPECT_THAT(storage->AddKeys(/*document_id=*/1, kDefaultSectionId, + /*new_keys=*/{201, 209, -149}), + IsOk()); + EXPECT_THAT(storage->AddKeys(/*document_id=*/2, kDefaultSectionId, + /*new_keys=*/{208}), + IsOk()); + EXPECT_THAT(storage->num_data(), Eq(6)); + + ICING_ASSERT_OK(storage->PersistToDisk()); + } + + // Check sorted_buckets manually. + const std::string sorted_buckets_file_path = absl_ports::StrCat( + working_path_, "/", IntegerIndexStorage::kFilePrefix, ".s"); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<FileBackedVector<Bucket>> sorted_buckets, + FileBackedVector<Bucket>::Create( + filesystem_, sorted_buckets_file_path, + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC)); + EXPECT_THAT(sorted_buckets->num_elements(), Eq(5)); + + ICING_ASSERT_OK_AND_ASSIGN(const Bucket* sbk1, + sorted_buckets->Get(/*idx=*/0)); + EXPECT_THAT(sbk1->key_lower(), Eq(-1000)); + EXPECT_THAT(sbk1->key_upper(), Eq(-100)); + EXPECT_THAT(sbk1->num_data(), Eq(2)); + ICING_ASSERT_OK_AND_ASSIGN(const Bucket* sbk2, + sorted_buckets->Get(/*idx=*/1)); + EXPECT_THAT(sbk2->key_lower(), Eq(0)); + EXPECT_THAT(sbk2->key_upper(), Eq(100)); + EXPECT_THAT(sbk2->num_data(), Eq(0)); + ICING_ASSERT_OK_AND_ASSIGN(const Bucket* sbk3, + sorted_buckets->Get(/*idx=*/2)); + EXPECT_THAT(sbk3->key_lower(), Eq(150)); + EXPECT_THAT(sbk3->key_upper(), Eq(199)); + EXPECT_THAT(sbk3->num_data(), Eq(0)); + ICING_ASSERT_OK_AND_ASSIGN(const Bucket* sbk4, + sorted_buckets->Get(/*idx=*/3)); + EXPECT_THAT(sbk4->key_lower(), Eq(200)); + EXPECT_THAT(sbk4->key_upper(), Eq(300)); + EXPECT_THAT(sbk4->num_data(), Eq(3)); + ICING_ASSERT_OK_AND_ASSIGN(const Bucket* sbk5, + sorted_buckets->Get(/*idx=*/4)); + EXPECT_THAT(sbk5->key_lower(), Eq(301)); + EXPECT_THAT(sbk5->key_upper(), Eq(999)); + EXPECT_THAT(sbk5->num_data(), Eq(0)); + + // Check unsorted_buckets and unsorted buckets manually. + const std::string unsorted_buckets_file_path = absl_ports::StrCat( + working_path_, "/", IntegerIndexStorage::kFilePrefix, ".u"); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<FileBackedVector<Bucket>> unsorted_buckets, + FileBackedVector<Bucket>::Create( + filesystem_, unsorted_buckets_file_path, + MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC)); + EXPECT_THAT(unsorted_buckets->num_elements(), Eq(4)); + + ICING_ASSERT_OK_AND_ASSIGN(const Bucket* ubk1, + unsorted_buckets->Get(/*idx=*/0)); + EXPECT_THAT(ubk1->key_lower(), Eq(1000)); + EXPECT_THAT(ubk1->key_upper(), Eq(std::numeric_limits<int64_t>::max())); + EXPECT_THAT(ubk1->num_data(), Eq(0)); + ICING_ASSERT_OK_AND_ASSIGN(const Bucket* ubk2, + unsorted_buckets->Get(/*idx=*/1)); + EXPECT_THAT(ubk2->key_lower(), Eq(-99)); + EXPECT_THAT(ubk2->key_upper(), Eq(-1)); + EXPECT_THAT(ubk2->num_data(), Eq(1)); + ICING_ASSERT_OK_AND_ASSIGN(const Bucket* ubk3, + unsorted_buckets->Get(/*idx=*/2)); + EXPECT_THAT(ubk3->key_lower(), Eq(101)); + EXPECT_THAT(ubk3->key_upper(), Eq(149)); + EXPECT_THAT(ubk3->num_data(), Eq(0)); + ICING_ASSERT_OK_AND_ASSIGN(const Bucket* ubk4, + unsorted_buckets->Get(/*idx=*/3)); + EXPECT_THAT(ubk4->key_lower(), Eq(std::numeric_limits<int64_t>::min())); + EXPECT_THAT(ubk4->key_upper(), Eq(-1001)); + EXPECT_THAT(ubk4->num_data(), Eq(0)); +} + TEST_P(IntegerIndexStorageTest, ExactQuerySortedBuckets) { // We use predefined custom buckets to initialize new integer index storage // and create some test keys accordingly. @@ -609,6 +788,7 @@ TEST_P(IntegerIndexStorageTest, ExactQuerySortedBuckets) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -664,6 +844,7 @@ TEST_P(IntegerIndexStorageTest, ExactQueryUnsortedBuckets) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -725,6 +906,7 @@ TEST_P(IntegerIndexStorageTest, ExactQueryIdenticalKeys) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -768,6 +950,7 @@ TEST_P(IntegerIndexStorageTest, RangeQueryEmptyIntegerIndexStorage) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -792,6 +975,7 @@ TEST_P(IntegerIndexStorageTest, RangeQuerySingleEntireSortedBucket) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -847,6 +1031,7 @@ TEST_P(IntegerIndexStorageTest, RangeQuerySingleEntireUnsortedBucket) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -905,6 +1090,7 @@ TEST_P(IntegerIndexStorageTest, RangeQuerySinglePartialSortedBucket) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -957,6 +1143,7 @@ TEST_P(IntegerIndexStorageTest, RangeQuerySinglePartialUnsortedBucket) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -1009,6 +1196,7 @@ TEST_P(IntegerIndexStorageTest, RangeQueryMultipleBuckets) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -1098,6 +1286,7 @@ TEST_P(IntegerIndexStorageTest, BatchAdd) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -1126,9 +1315,11 @@ TEST_P(IntegerIndexStorageTest, BatchAdd) { TEST_P(IntegerIndexStorageTest, BatchAddShouldDedupeKeys) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); std::vector<int64_t> keys = {2, 3, 1, 2, 4, -1, -1, 100, 3}; EXPECT_THAT( @@ -1152,6 +1343,7 @@ TEST_P(IntegerIndexStorageTest, MultipleKeysShouldMergeAndDedupeDocHitInfo) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -1188,6 +1380,7 @@ TEST_P(IntegerIndexStorageTest, filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -1236,25 +1429,26 @@ TEST_P(IntegerIndexStorageTest, } TEST_P(IntegerIndexStorageTest, SplitBuckets) { + int32_t custom_num_data_threshold_for_bucket_split = 300; + ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); - - uint32_t block_size = FlashIndexStorage::SelectBlockSize(); - uint32_t max_posting_list_bytes = IndexBlock::CalculateMaxPostingListBytes( - block_size, serializer_->GetDataTypeBytes()); - uint32_t max_num_data_before_split = - max_posting_list_bytes / serializer_->GetDataTypeBytes(); - - // Add max_num_data_before_split + 1 keys to invoke bucket splitting. - // Keys: max_num_data_before_split to 0 - // Document ids: 0 to max_num_data_before_split + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(/*custom_init_sorted_buckets_in=*/{}, + /*custom_init_unsorted_buckets_in=*/{}, + custom_num_data_threshold_for_bucket_split, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); + + // Add custom_num_data_threshold_for_bucket_split + 1 keys to invoke bucket + // splitting. + // - Keys: custom_num_data_threshold_for_bucket_split to 0 Document + // - ids: 0 to custom_num_data_threshold_for_bucket_split std::unordered_map<int64_t, DocumentId> data; - int64_t key = max_num_data_before_split; + int64_t key = custom_num_data_threshold_for_bucket_split; DocumentId document_id = 0; - for (int i = 0; i < max_num_data_before_split + 1; ++i) { + for (int i = 0; i < custom_num_data_threshold_for_bucket_split + 1; ++i) { data[key] = document_id; ICING_ASSERT_OK( storage->AddKeys(document_id, kDefaultSectionId, /*new_keys=*/{key})); @@ -1299,7 +1493,8 @@ TEST_P(IntegerIndexStorageTest, SplitBuckets) { // Ensure that search works normally. std::vector<SectionId> expected_sections = {kDefaultSectionId}; - for (int64_t key = max_num_data_before_split; key >= 0; key--) { + for (int64_t key = custom_num_data_threshold_for_bucket_split; key >= 0; + key--) { ASSERT_THAT(data, Contains(Key(key))); DocumentId expected_document_id = data[key]; EXPECT_THAT(Query(storage.get(), /*key_lower=*/key, /*key_upper=*/key), @@ -1309,20 +1504,21 @@ TEST_P(IntegerIndexStorageTest, SplitBuckets) { } TEST_P(IntegerIndexStorageTest, SplitBucketsTriggerSortBuckets) { + int32_t custom_num_data_threshold_for_bucket_split = 300; + ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); - - uint32_t block_size = FlashIndexStorage::SelectBlockSize(); - uint32_t max_posting_list_bytes = IndexBlock::CalculateMaxPostingListBytes( - block_size, serializer_->GetDataTypeBytes()); - uint32_t max_num_data_before_split = - max_posting_list_bytes / serializer_->GetDataTypeBytes(); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(/*custom_init_sorted_buckets_in=*/{}, + /*custom_init_unsorted_buckets_in=*/{}, + custom_num_data_threshold_for_bucket_split, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); // Add IntegerIndexStorage::kUnsortedBucketsLengthThreshold keys. For each - // key, add max_num_data_before_split + 1 data. Then we will get: + // key, add custom_num_data_threshold_for_bucket_split + 1 data. Then we will + // get: // - Bucket splitting will create kUnsortedBucketsLengthThreshold + 1 unsorted // buckets [[50, 50], [49, 49], ..., [1, 1], [51, INT64_MAX]]. // - Since there are kUnsortedBucketsLengthThreshold + 1 unsorted buckets, we @@ -1332,7 +1528,7 @@ TEST_P(IntegerIndexStorageTest, SplitBucketsTriggerSortBuckets) { DocumentId document_id = 0; for (int i = 0; i < IntegerIndexStorage::kUnsortedBucketsLengthThreshold; ++i) { - for (int j = 0; j < max_num_data_before_split + 1; ++j) { + for (int j = 0; j < custom_num_data_threshold_for_bucket_split + 1; ++j) { data[key].push_back(document_id); ICING_ASSERT_OK( storage->AddKeys(document_id, kDefaultSectionId, /*new_keys=*/{key})); @@ -1396,6 +1592,7 @@ TEST_P(IntegerIndexStorageTest, TransferIndex) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -1433,9 +1630,11 @@ TEST_P(IntegerIndexStorageTest, TransferIndex) { { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> new_storage, - IntegerIndexStorage::Create(filesystem_, working_path_ + "_temp", - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_ + "_temp", + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); EXPECT_THAT( storage->TransferIndex(document_id_old_to_new, new_storage.get()), IsOk()); @@ -1445,9 +1644,11 @@ TEST_P(IntegerIndexStorageTest, TransferIndex) { // Verify after transferring and reinitializing the instance. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> new_storage, - IntegerIndexStorage::Create(filesystem_, working_path_ + "_temp", - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_ + "_temp", + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); std::vector<SectionId> expected_sections = {kDefaultSectionId}; EXPECT_THAT(new_storage->num_data(), Eq(7)); @@ -1493,9 +1694,11 @@ TEST_P(IntegerIndexStorageTest, TransferIndex) { TEST_P(IntegerIndexStorageTest, TransferIndexOutOfRangeDocumentId) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create(filesystem_, working_path_, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_, + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); ICING_ASSERT_OK(storage->AddKeys(/*document_id=*/1, kDefaultSectionId, /*new_keys=*/{120})); @@ -1510,9 +1713,11 @@ TEST_P(IntegerIndexStorageTest, TransferIndexOutOfRangeDocumentId) { // Transfer to new storage. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> new_storage, - IntegerIndexStorage::Create(filesystem_, working_path_ + "_temp", - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_ + "_temp", + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); EXPECT_THAT(storage->TransferIndex(document_id_old_to_new, new_storage.get()), IsOk()); @@ -1542,6 +1747,7 @@ TEST_P(IntegerIndexStorageTest, TransferEmptyIndex) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); ASSERT_THAT(storage->num_data(), Eq(0)); @@ -1552,9 +1758,11 @@ TEST_P(IntegerIndexStorageTest, TransferEmptyIndex) { // Transfer to new storage. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> new_storage, - IntegerIndexStorage::Create(filesystem_, working_path_ + "_temp", - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_ + "_temp", + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); EXPECT_THAT(storage->TransferIndex(document_id_old_to_new, new_storage.get()), IsOk()); @@ -1581,6 +1789,7 @@ TEST_P(IntegerIndexStorageTest, TransferIndexDeleteAll) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -1605,9 +1814,11 @@ TEST_P(IntegerIndexStorageTest, TransferIndexDeleteAll) { { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> new_storage, - IntegerIndexStorage::Create(filesystem_, working_path_ + "_temp", - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_ + "_temp", + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); EXPECT_THAT( storage->TransferIndex(document_id_old_to_new, new_storage.get()), IsOk()); @@ -1617,9 +1828,11 @@ TEST_P(IntegerIndexStorageTest, TransferIndexDeleteAll) { // Verify after transferring and reinitializing the instance. ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> new_storage, - IntegerIndexStorage::Create(filesystem_, working_path_ + "_temp", - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, working_path_ + "_temp", + Options(IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); std::vector<SectionId> expected_sections = {kDefaultSectionId}; EXPECT_THAT(new_storage->num_data(), Eq(0)); @@ -1630,6 +1843,11 @@ TEST_P(IntegerIndexStorageTest, TransferIndexDeleteAll) { } TEST_P(IntegerIndexStorageTest, TransferIndexShouldInvokeMergeBuckets) { + int32_t custom_num_data_threshold_for_bucket_split = 300; + int32_t custom_num_data_threshold_for_bucket_merge = + IntegerIndexStorage::kNumDataThresholdRatioForBucketMerge * + custom_num_data_threshold_for_bucket_split; + // This test verifies that if TransferIndex invokes bucket merging logic to // ensure sure we're able to avoid having mostly empty buckets after inserting // and deleting data for many rounds. @@ -1648,6 +1866,7 @@ TEST_P(IntegerIndexStorageTest, TransferIndexShouldInvokeMergeBuckets) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + custom_num_data_threshold_for_bucket_split, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); @@ -1671,7 +1890,7 @@ TEST_P(IntegerIndexStorageTest, TransferIndexShouldInvokeMergeBuckets) { /*new_keys=*/{20})); ASSERT_THAT(storage->num_data(), Eq(9)); ASSERT_THAT(storage->num_data(), - Le(IntegerIndexStorage::kNumDataThresholdForBucketMerge)); + Le(custom_num_data_threshold_for_bucket_merge)); // Create document_id_old_to_new that keeps all existing documents. std::vector<DocumentId> document_id_old_to_new(9); @@ -1683,12 +1902,17 @@ TEST_P(IntegerIndexStorageTest, TransferIndexShouldInvokeMergeBuckets) { { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> new_storage, - IntegerIndexStorage::Create(filesystem_, new_storage_working_path, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, new_storage_working_path, + Options(/*custom_init_sorted_buckets_in=*/{}, + /*custom_init_unsorted_buckets_in=*/{}, + custom_num_data_threshold_for_bucket_split, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); EXPECT_THAT( storage->TransferIndex(document_id_old_to_new, new_storage.get()), IsOk()); + EXPECT_THAT(new_storage->num_data(), Eq(9)); } // Check new_storage->sorted_bucket_ manually. @@ -1704,9 +1928,15 @@ TEST_P(IntegerIndexStorageTest, TransferIndexShouldInvokeMergeBuckets) { ICING_ASSERT_OK_AND_ASSIGN(const Bucket* bk1, sorted_buckets->Get(/*idx=*/0)); EXPECT_THAT(bk1->key_lower(), Eq(std::numeric_limits<int64_t>::min())); EXPECT_THAT(bk1->key_upper(), Eq(std::numeric_limits<int64_t>::max())); + EXPECT_THAT(bk1->num_data(), Eq(9)); } TEST_P(IntegerIndexStorageTest, TransferIndexExceedsMergeThreshold) { + int32_t custom_num_data_threshold_for_bucket_split = 300; + int32_t custom_num_data_threshold_for_bucket_merge = + IntegerIndexStorage::kNumDataThresholdRatioForBucketMerge * + custom_num_data_threshold_for_bucket_split; + // This test verifies that if TransferIndex invokes bucket merging logic and // doesn't merge buckets too aggressively to ensure we won't get a bucket with // too many data. @@ -1725,15 +1955,16 @@ TEST_P(IntegerIndexStorageTest, TransferIndexExceedsMergeThreshold) { filesystem_, working_path_, Options(std::move(custom_init_sorted_buckets), std::move(custom_init_unsorted_buckets), + custom_num_data_threshold_for_bucket_split, /*pre_mapping_fbv_in=*/GetParam()), serializer_.get())); // Insert data into 2 buckets so that total # of these 2 buckets exceed - // kNumDataThresholdForBucketMerge. + // custom_num_data_threshold_for_bucket_merge. // - Bucket 1: [-1000, -100] // - Bucket 2: [101, 149] DocumentId document_id = 0; - int num_data_for_bucket1 = 200; + int num_data_for_bucket1 = custom_num_data_threshold_for_bucket_merge - 50; for (int i = 0; i < num_data_for_bucket1; ++i) { ICING_ASSERT_OK(storage->AddKeys(document_id, kDefaultSectionId, /*new_keys=*/{-200})); @@ -1747,8 +1978,10 @@ TEST_P(IntegerIndexStorageTest, TransferIndexExceedsMergeThreshold) { ++document_id; } + ASSERT_THAT(storage->num_data(), + Eq(num_data_for_bucket1 + num_data_for_bucket2)); ASSERT_THAT(num_data_for_bucket1 + num_data_for_bucket2, - Gt(IntegerIndexStorage::kNumDataThresholdForBucketMerge)); + Gt(custom_num_data_threshold_for_bucket_merge)); // Create document_id_old_to_new that keeps all existing documents. std::vector<DocumentId> document_id_old_to_new(document_id); @@ -1760,12 +1993,18 @@ TEST_P(IntegerIndexStorageTest, TransferIndexExceedsMergeThreshold) { { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndexStorage> new_storage, - IntegerIndexStorage::Create(filesystem_, new_storage_working_path, - Options(/*pre_mapping_fbv_in=*/GetParam()), - serializer_.get())); + IntegerIndexStorage::Create( + filesystem_, new_storage_working_path, + Options(/*custom_init_sorted_buckets_in=*/{}, + /*custom_init_unsorted_buckets_in=*/{}, + custom_num_data_threshold_for_bucket_split, + /*pre_mapping_fbv_in=*/GetParam()), + serializer_.get())); EXPECT_THAT( storage->TransferIndex(document_id_old_to_new, new_storage.get()), IsOk()); + EXPECT_THAT(new_storage->num_data(), + Eq(num_data_for_bucket1 + num_data_for_bucket2)); } // Check new_storage->sorted_bucket_ manually. @@ -1781,9 +2020,11 @@ TEST_P(IntegerIndexStorageTest, TransferIndexExceedsMergeThreshold) { ICING_ASSERT_OK_AND_ASSIGN(const Bucket* bk1, sorted_buckets->Get(/*idx=*/0)); EXPECT_THAT(bk1->key_lower(), Eq(std::numeric_limits<int64_t>::min())); EXPECT_THAT(bk1->key_upper(), Eq(100)); + EXPECT_THAT(bk1->num_data(), Eq(num_data_for_bucket1)); ICING_ASSERT_OK_AND_ASSIGN(const Bucket* bk2, sorted_buckets->Get(/*idx=*/1)); EXPECT_THAT(bk2->key_lower(), Eq(101)); EXPECT_THAT(bk2->key_upper(), Eq(std::numeric_limits<int64_t>::max())); + EXPECT_THAT(bk2->num_data(), Eq(num_data_for_bucket2)); } INSTANTIATE_TEST_SUITE_P(IntegerIndexStorageTest, IntegerIndexStorageTest, diff --git a/icing/index/numeric/integer-index.cc b/icing/index/numeric/integer-index.cc index 5fa82a5..b2fe159 100644 --- a/icing/index/numeric/integer-index.cc +++ b/icing/index/numeric/integer-index.cc @@ -91,7 +91,7 @@ libtextclassifier3::StatusOr<IntegerIndex::PropertyToStorageMapType> GetPropertyIntegerIndexStorageMap( const Filesystem& filesystem, const std::string& working_path, PostingListIntegerIndexSerializer* posting_list_serializer, - bool pre_mapping_fbv) { + int32_t num_data_threshold_for_bucket_split, bool pre_mapping_fbv) { ICING_ASSIGN_OR_RETURN(std::vector<std::string> property_paths, GetAllExistingPropertyPaths(filesystem, working_path)); @@ -102,11 +102,13 @@ GetPropertyIntegerIndexStorageMap( } std::string storage_working_path = GetPropertyIndexStoragePath(working_path, property_path); - ICING_ASSIGN_OR_RETURN(std::unique_ptr<IntegerIndexStorage> storage, - IntegerIndexStorage::Create( - filesystem, storage_working_path, - IntegerIndexStorage::Options(pre_mapping_fbv), - posting_list_serializer)); + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<IntegerIndexStorage> storage, + IntegerIndexStorage::Create( + filesystem, storage_working_path, + IntegerIndexStorage::Options(num_data_threshold_for_bucket_split, + pre_mapping_fbv), + posting_list_serializer)); property_to_storage_map.insert( std::make_pair(property_path, std::move(storage))); } @@ -141,6 +143,8 @@ libtextclassifier3::StatusOr<std::unordered_set<std::string>> CreatePropertySet( } // namespace libtextclassifier3::Status IntegerIndex::Editor::IndexAllBufferedKeys() && { + integer_index_.SetDirty(); + auto iter = integer_index_.property_to_storage_map_.find(property_path_); IntegerIndexStorage* target_storage = nullptr; // 1. Check if this property already has its own individual index. @@ -161,7 +165,8 @@ libtextclassifier3::Status IntegerIndex::Editor::IndexAllBufferedKeys() && { integer_index_.filesystem_, GetPropertyIndexStoragePath(integer_index_.working_path_, kWildcardPropertyIndexFileName), - IntegerIndexStorage::Options(pre_mapping_fbv_), + IntegerIndexStorage::Options(num_data_threshold_for_bucket_split_, + pre_mapping_fbv_), integer_index_.posting_list_serializer_.get())); } ICING_RETURN_IF_ERROR( @@ -175,7 +180,8 @@ libtextclassifier3::Status IntegerIndex::Editor::IndexAllBufferedKeys() && { integer_index_.filesystem_, GetPropertyIndexStoragePath(integer_index_.working_path_, property_path_), - IntegerIndexStorage::Options(pre_mapping_fbv_), + IntegerIndexStorage::Options(num_data_threshold_for_bucket_split_, + pre_mapping_fbv_), integer_index_.posting_list_serializer_.get())); target_storage = new_storage.get(); integer_index_.property_to_storage_map_.insert( @@ -188,6 +194,7 @@ libtextclassifier3::Status IntegerIndex::Editor::IndexAllBufferedKeys() && { /* static */ libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndex>> IntegerIndex::Create(const Filesystem& filesystem, std::string working_path, + int32_t num_data_threshold_for_bucket_split, bool pre_mapping_fbv) { if (!filesystem.FileExists(GetMetadataFilePath(working_path).c_str())) { // Discard working_path if metadata file is missing, and reinitialize. @@ -195,9 +202,11 @@ IntegerIndex::Create(const Filesystem& filesystem, std::string working_path, ICING_RETURN_IF_ERROR(Discard(filesystem, working_path)); } return InitializeNewFiles(filesystem, std::move(working_path), + num_data_threshold_for_bucket_split, pre_mapping_fbv); } return InitializeExistingFiles(filesystem, std::move(working_path), + num_data_threshold_for_bucket_split, pre_mapping_fbv); } @@ -239,6 +248,8 @@ IntegerIndex::GetIterator(std::string_view property_path, int64_t key_lower, libtextclassifier3::Status IntegerIndex::AddPropertyToWildcardStorage( const std::string& property_path) { + SetDirty(); + WildcardPropertyStorage wildcard_properties; wildcard_properties.mutable_property_entries()->Reserve( wildcard_properties_set_.size()); @@ -272,7 +283,8 @@ libtextclassifier3::Status IntegerIndex::Optimize( // we can safely swap directories later. ICING_ASSIGN_OR_RETURN( std::unique_ptr<IntegerIndex> new_integer_index, - Create(filesystem_, temp_working_path_ddir.dir(), pre_mapping_fbv_)); + Create(filesystem_, temp_working_path_ddir.dir(), + num_data_threshold_for_bucket_split_, pre_mapping_fbv_)); ICING_RETURN_IF_ERROR( TransferIndex(document_id_old_to_new, new_integer_index.get())); new_integer_index->set_last_added_document_id(new_last_added_document_id); @@ -322,20 +334,24 @@ libtextclassifier3::Status IntegerIndex::Optimize( filesystem_, GetPropertyIndexStoragePath(working_path_, kWildcardPropertyIndexFileName), - IntegerIndexStorage::Options(pre_mapping_fbv_), + IntegerIndexStorage::Options(num_data_threshold_for_bucket_split_, + pre_mapping_fbv_), posting_list_serializer_.get())); } // Initialize all existing integer index storages. - ICING_ASSIGN_OR_RETURN(property_to_storage_map_, - GetPropertyIntegerIndexStorageMap( - filesystem_, working_path_, - posting_list_serializer_.get(), pre_mapping_fbv_)); + ICING_ASSIGN_OR_RETURN( + property_to_storage_map_, + GetPropertyIntegerIndexStorageMap( + filesystem_, working_path_, posting_list_serializer_.get(), + num_data_threshold_for_bucket_split_, pre_mapping_fbv_)); return libtextclassifier3::Status::OK; } libtextclassifier3::Status IntegerIndex::Clear() { + SetDirty(); + // Step 1: clear property_to_storage_map_. property_to_storage_map_.clear(); wildcard_index_storage_.reset(); @@ -367,6 +383,7 @@ libtextclassifier3::Status IntegerIndex::Clear() { /* static */ libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndex>> IntegerIndex::InitializeNewFiles(const Filesystem& filesystem, std::string&& working_path, + int32_t num_data_threshold_for_bucket_split, bool pre_mapping_fbv) { // Create working directory. if (!filesystem.CreateDirectoryRecursively(working_path.c_str())) { @@ -399,12 +416,14 @@ IntegerIndex::InitializeNewFiles(const Filesystem& filesystem, std::make_unique<MemoryMappedFile>(std::move(metadata_mmapped_file)), /*property_to_storage_map=*/{}, std::move(wildcard_property_storage), /*wildcard_properties_set=*/{}, /*wildcard_index_storage=*/nullptr, - pre_mapping_fbv)); + num_data_threshold_for_bucket_split, pre_mapping_fbv)); // Initialize info content by writing mapped memory directly. Info& info_ref = new_integer_index->info(); info_ref.magic = Info::kMagic; info_ref.last_added_document_id = kInvalidDocumentId; + info_ref.num_data_threshold_for_bucket_split = + num_data_threshold_for_bucket_split; // Initialize new PersistentStorage. The initial checksums will be computed // and set via InitializeNewStorage. ICING_RETURN_IF_ERROR(new_integer_index->InitializeNewStorage()); @@ -413,9 +432,9 @@ IntegerIndex::InitializeNewFiles(const Filesystem& filesystem, } /* static */ libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndex>> -IntegerIndex::InitializeExistingFiles(const Filesystem& filesystem, - std::string&& working_path, - bool pre_mapping_fbv) { +IntegerIndex::InitializeExistingFiles( + const Filesystem& filesystem, std::string&& working_path, + int32_t num_data_threshold_for_bucket_split, bool pre_mapping_fbv) { // Mmap the content of the crcs and info. ICING_ASSIGN_OR_RETURN( MemoryMappedFile metadata_mmapped_file, @@ -432,10 +451,11 @@ IntegerIndex::InitializeExistingFiles(const Filesystem& filesystem, std::make_unique<PostingListIntegerIndexSerializer>(); // Initialize all existing integer index storages. - ICING_ASSIGN_OR_RETURN(PropertyToStorageMapType property_to_storage_map, - GetPropertyIntegerIndexStorageMap( - filesystem, working_path, - posting_list_serializer.get(), pre_mapping_fbv)); + ICING_ASSIGN_OR_RETURN( + PropertyToStorageMapType property_to_storage_map, + GetPropertyIntegerIndexStorageMap( + filesystem, working_path, posting_list_serializer.get(), + num_data_threshold_for_bucket_split, pre_mapping_fbv)); std::string wildcard_property_path = GetWildcardPropertyStorageFilePath(working_path); @@ -455,7 +475,8 @@ IntegerIndex::InitializeExistingFiles(const Filesystem& filesystem, filesystem, GetPropertyIndexStoragePath(working_path, kWildcardPropertyIndexFileName), - IntegerIndexStorage::Options(pre_mapping_fbv), + IntegerIndexStorage::Options(num_data_threshold_for_bucket_split, + pre_mapping_fbv), posting_list_serializer.get())); } @@ -465,7 +486,7 @@ IntegerIndex::InitializeExistingFiles(const Filesystem& filesystem, std::make_unique<MemoryMappedFile>(std::move(metadata_mmapped_file)), std::move(property_to_storage_map), std::move(wildcard_property_storage), std::move(wildcard_properties_set), std::move(wildcard_index_storage), - pre_mapping_fbv)); + num_data_threshold_for_bucket_split, pre_mapping_fbv)); // Initialize existing PersistentStorage. Checksums will be validated. ICING_RETURN_IF_ERROR(integer_index->InitializeExistingStorage()); @@ -474,6 +495,14 @@ IntegerIndex::InitializeExistingFiles(const Filesystem& filesystem, return absl_ports::FailedPreconditionError("Incorrect magic value"); } + // If num_data_threshold_for_bucket_split mismatches, then return error to let + // caller rebuild. + if (integer_index->info().num_data_threshold_for_bucket_split != + num_data_threshold_for_bucket_split) { + return absl_ports::FailedPreconditionError( + "Mismatch num_data_threshold_for_bucket_split"); + } + return integer_index; } @@ -488,7 +517,8 @@ IntegerIndex::TransferIntegerIndexStorage( std::unique_ptr<IntegerIndexStorage> new_storage, IntegerIndexStorage::Create( new_integer_index->filesystem_, new_storage_working_path, - IntegerIndexStorage::Options(pre_mapping_fbv_), + IntegerIndexStorage::Options(num_data_threshold_for_bucket_split_, + pre_mapping_fbv_), new_integer_index->posting_list_serializer_.get())); ICING_RETURN_IF_ERROR( @@ -552,7 +582,11 @@ libtextclassifier3::Status IntegerIndex::TransferIndex( return libtextclassifier3::Status::OK; } -libtextclassifier3::Status IntegerIndex::PersistStoragesToDisk() { +libtextclassifier3::Status IntegerIndex::PersistStoragesToDisk(bool force) { + if (!force && !is_storage_dirty()) { + return libtextclassifier3::Status::OK; + } + for (auto& [_, storage] : property_to_storage_map_) { ICING_RETURN_IF_ERROR(storage->PersistToDisk()); } @@ -564,18 +598,32 @@ libtextclassifier3::Status IntegerIndex::PersistStoragesToDisk() { return libtextclassifier3::Status::OK; } -libtextclassifier3::Status IntegerIndex::PersistMetadataToDisk() { +libtextclassifier3::Status IntegerIndex::PersistMetadataToDisk(bool force) { + if (!force && !is_info_dirty() && !is_storage_dirty()) { + return libtextclassifier3::Status::OK; + } + // Changes should have been applied to the underlying file when using // MemoryMappedFile::Strategy::READ_WRITE_AUTO_SYNC, but call msync() as an // extra safety step to ensure they are written out. return metadata_mmapped_file_->PersistToDisk(); } -libtextclassifier3::StatusOr<Crc32> IntegerIndex::ComputeInfoChecksum() { +libtextclassifier3::StatusOr<Crc32> IntegerIndex::ComputeInfoChecksum( + bool force) { + if (!force && !is_info_dirty()) { + return Crc32(crcs().component_crcs.info_crc); + } + return info().ComputeChecksum(); } -libtextclassifier3::StatusOr<Crc32> IntegerIndex::ComputeStoragesChecksum() { +libtextclassifier3::StatusOr<Crc32> IntegerIndex::ComputeStoragesChecksum( + bool force) { + if (!force && !is_storage_dirty()) { + return Crc32(crcs().component_crcs.storages_crc); + } + // XOR all crcs of all storages. Since XOR is commutative and associative, // the order doesn't matter. uint32_t storages_checksum = 0; diff --git a/icing/index/numeric/integer-index.h b/icing/index/numeric/integer-index.h index 30f9852..e7a3127 100644 --- a/icing/index/numeric/integer-index.h +++ b/icing/index/numeric/integer-index.h @@ -55,25 +55,29 @@ class IntegerIndex : public NumericIndex<int64_t> { // 'wildcard' storage. static constexpr int kMaxPropertyStorages = 32; + static constexpr int32_t kDefaultNumDataThresholdForBucketSplit = + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit; + struct Info { - static constexpr int32_t kMagic = 0x238a3dcb; + static constexpr int32_t kMagic = 0x5d8a1e8a; int32_t magic; DocumentId last_added_document_id; + int32_t num_data_threshold_for_bucket_split; Crc32 ComputeChecksum() const { return Crc32( std::string_view(reinterpret_cast<const char*>(this), sizeof(Info))); } } __attribute__((packed)); - static_assert(sizeof(Info) == 8, ""); + static_assert(sizeof(Info) == 12, ""); // Metadata file layout: <Crcs><Info> static constexpr int32_t kCrcsMetadataFileOffset = 0; static constexpr int32_t kInfoMetadataFileOffset = static_cast<int32_t>(sizeof(Crcs)); static constexpr int32_t kMetadataFileSize = sizeof(Crcs) + sizeof(Info); - static_assert(kMetadataFileSize == 20, ""); + static_assert(kMetadataFileSize == 24, ""); static constexpr WorkingPathType kWorkingPathType = WorkingPathType::kDirectory; @@ -90,6 +94,8 @@ class IntegerIndex : public NumericIndex<int64_t> { // related files will be stored under this directory. See // PersistentStorage for more details about the concept of // working_path. + // num_data_threshold_for_bucket_split: see IntegerIndexStorage::Options for + // more details. // pre_mapping_fbv: flag indicating whether memory map max possible file size // for underlying FileBackedVector before growing the actual // file size. @@ -101,7 +107,7 @@ class IntegerIndex : public NumericIndex<int64_t> { // - Any FileBackedVector/MemoryMappedFile errors. static libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndex>> Create( const Filesystem& filesystem, std::string working_path, - bool pre_mapping_fbv); + int32_t num_data_threshold_for_bucket_split, bool pre_mapping_fbv); // Deletes IntegerIndex under working_path. // @@ -122,7 +128,8 @@ class IntegerIndex : public NumericIndex<int64_t> { std::string_view property_path, DocumentId document_id, SectionId section_id) override { return std::make_unique<Editor>(property_path, document_id, section_id, - *this, pre_mapping_fbv_); + *this, num_data_threshold_for_bucket_split_, + pre_mapping_fbv_); } // Returns a DocHitInfoIterator for iterating through all docs which have the @@ -172,6 +179,8 @@ class IntegerIndex : public NumericIndex<int64_t> { } void set_last_added_document_id(DocumentId document_id) override { + SetInfoDirty(); + Info& info_ref = info(); if (info_ref.last_added_document_id == kInvalidDocumentId || document_id > info_ref.last_added_document_id) { @@ -189,9 +198,12 @@ class IntegerIndex : public NumericIndex<int64_t> { public: explicit Editor(std::string_view property_path, DocumentId document_id, SectionId section_id, IntegerIndex& integer_index, + int32_t num_data_threshold_for_bucket_split, bool pre_mapping_fbv) : NumericIndex<int64_t>::Editor(property_path, document_id, section_id), integer_index_(integer_index), + num_data_threshold_for_bucket_split_( + num_data_threshold_for_bucket_split), pre_mapping_fbv_(pre_mapping_fbv) {} ~Editor() override = default; @@ -211,6 +223,8 @@ class IntegerIndex : public NumericIndex<int64_t> { IntegerIndex& integer_index_; // Does not own. + int32_t num_data_threshold_for_bucket_split_; + // Flag indicating whether memory map max possible file size for underlying // FileBackedVector before growing the actual file size. bool pre_mapping_fbv_; @@ -226,7 +240,7 @@ class IntegerIndex : public NumericIndex<int64_t> { wildcard_property_storage, std::unordered_set<std::string> wildcard_properties_set, std::unique_ptr<icing::lib::IntegerIndexStorage> wildcard_index_storage, - bool pre_mapping_fbv) + int32_t num_data_threshold_for_bucket_split, bool pre_mapping_fbv) : NumericIndex<int64_t>(filesystem, std::move(working_path), kWorkingPathType), posting_list_serializer_(std::move(posting_list_serializer)), @@ -235,15 +249,22 @@ class IntegerIndex : public NumericIndex<int64_t> { wildcard_property_storage_(std::move(wildcard_property_storage)), wildcard_properties_set_(std::move(wildcard_properties_set)), wildcard_index_storage_(std::move(wildcard_index_storage)), - pre_mapping_fbv_(pre_mapping_fbv) {} + num_data_threshold_for_bucket_split_( + num_data_threshold_for_bucket_split), + pre_mapping_fbv_(pre_mapping_fbv), + is_info_dirty_(false), + is_storage_dirty_(false) {} static libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndex>> InitializeNewFiles(const Filesystem& filesystem, std::string&& working_path, + int32_t num_data_threshold_for_bucket_split, bool pre_mapping_fbv); static libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndex>> InitializeExistingFiles(const Filesystem& filesystem, - std::string&& working_path, bool pre_mapping_fbv); + std::string&& working_path, + int32_t num_data_threshold_for_bucket_split, + bool pre_mapping_fbv); // Adds the property path to the list of properties using wildcard storage. // This will both update the in-memory list (wildcard_properties_set_) and @@ -296,20 +317,20 @@ class IntegerIndex : public NumericIndex<int64_t> { // Returns: // - OK on success // - INTERNAL_ERROR on I/O error - libtextclassifier3::Status PersistStoragesToDisk() override; + libtextclassifier3::Status PersistStoragesToDisk(bool force) override; // Flushes contents of metadata file. // // Returns: // - OK on success // - INTERNAL_ERROR on I/O error - libtextclassifier3::Status PersistMetadataToDisk() override; + libtextclassifier3::Status PersistMetadataToDisk(bool force) override; // Computes and returns Info checksum. // // Returns: // - Crc of the Info on success - libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum() override; + libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum(bool force) override; // Computes and returns all storages checksum. Checksums of (storage_crc, // property_path) for all existing property paths will be combined together by @@ -318,7 +339,8 @@ class IntegerIndex : public NumericIndex<int64_t> { // Returns: // - Crc of all storages on success // - INTERNAL_ERROR if any data inconsistency - libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum() override; + libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum( + bool force) override; Crcs& crcs() override { return *reinterpret_cast<Crcs*>(metadata_mmapped_file_->mutable_region() + @@ -340,6 +362,17 @@ class IntegerIndex : public NumericIndex<int64_t> { kInfoMetadataFileOffset); } + void SetInfoDirty() { is_info_dirty_ = true; } + // When storage is dirty, we have to set info dirty as well. So just expose + // SetDirty to set both. + void SetDirty() { + is_info_dirty_ = true; + is_storage_dirty_ = true; + } + + bool is_info_dirty() const { return is_info_dirty_; } + bool is_storage_dirty() const { return is_storage_dirty_; } + std::unique_ptr<PostingListIntegerIndexSerializer> posting_list_serializer_; std::unique_ptr<MemoryMappedFile> metadata_mmapped_file_; @@ -360,9 +393,14 @@ class IntegerIndex : public NumericIndex<int64_t> { // kMaxPropertyStorages in property_to_storage_map. std::unique_ptr<icing::lib::IntegerIndexStorage> wildcard_index_storage_; + int32_t num_data_threshold_for_bucket_split_; + // Flag indicating whether memory map max possible file size for underlying // FileBackedVector before growing the actual file size. bool pre_mapping_fbv_; + + bool is_info_dirty_; + bool is_storage_dirty_; }; } // namespace lib diff --git a/icing/index/numeric/integer-index_test.cc b/icing/index/numeric/integer-index_test.cc index 8a7acb9..b2e3fbe 100644 --- a/icing/index/numeric/integer-index_test.cc +++ b/icing/index/numeric/integer-index_test.cc @@ -83,13 +83,14 @@ class NumericIndexIntegerTest : public ::testing::Test { filesystem_.CreateDirectoryRecursively(document_store_dir.c_str())); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult doc_store_create_result, - DocumentStore::Create(&filesystem_, document_store_dir, &clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, document_store_dir, &clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); doc_store_ = std::move(doc_store_create_result.document_store); } @@ -114,8 +115,10 @@ class NumericIndexIntegerTest : public ::testing::Test { template <> libtextclassifier3::StatusOr<std::unique_ptr<NumericIndex<int64_t>>> CreateIntegerIndex<IntegerIndex>() { - return IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/false); + return IntegerIndex::Create( + filesystem_, working_path_, /*num_data_threshold_for_bucket_split=*/ + IntegerIndexStorage::kDefaultNumDataThresholdForBucketSplit, + /*pre_mapping_fbv=*/false); } template <typename NotIntegerIndexType> @@ -138,8 +141,7 @@ class NumericIndexIntegerTest : public ::testing::Test { } ICING_ASSIGN_OR_RETURN( std::vector<DocumentId> docid_map, - doc_store_->OptimizeInto(document_store_compact_dir, nullptr, - /*namespace_id_fingerprint=*/false)); + doc_store_->OptimizeInto(document_store_compact_dir, nullptr)); doc_store_.reset(); if (!filesystem_.SwapFiles(document_store_dir.c_str(), @@ -153,13 +155,14 @@ class NumericIndexIntegerTest : public ::testing::Test { ICING_ASSIGN_OR_RETURN( DocumentStore::CreateResult doc_store_create_result, - DocumentStore::Create(&filesystem_, document_store_dir, &clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, document_store_dir, &clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); doc_store_ = std::move(doc_store_create_result.document_store); return docid_map; } @@ -1126,14 +1129,28 @@ TYPED_TEST(NumericIndexIntegerTest, Clear) { /*document_id=*/4, std::vector<SectionId>{kDefaultSectionId})))); } +struct IntegerIndexTestParam { + int32_t num_data_threshold_for_bucket_split; + bool pre_mapping_fbv; + + explicit IntegerIndexTestParam(int32_t num_data_threshold_for_bucket_split_in, + bool pre_mapping_fbv_in) + : num_data_threshold_for_bucket_split( + num_data_threshold_for_bucket_split_in), + pre_mapping_fbv(pre_mapping_fbv_in) {} +}; + // Tests for persistent integer index only -class IntegerIndexTest : public NumericIndexIntegerTest<IntegerIndex>, - public ::testing::WithParamInterface<bool> {}; +class IntegerIndexTest + : public NumericIndexIntegerTest<IntegerIndex>, + public ::testing::WithParamInterface<IntegerIndexTestParam> {}; TEST_P(IntegerIndexTest, InvalidWorkingPath) { - EXPECT_THAT(IntegerIndex::Create(filesystem_, "/dev/null/integer_index_test", - /*pre_mapping_fbv=*/GetParam()), - StatusIs(libtextclassifier3::StatusCode::INTERNAL)); + EXPECT_THAT( + IntegerIndex::Create(filesystem_, "/dev/null/integer_index_test", + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv), + StatusIs(libtextclassifier3::StatusCode::INTERNAL)); } TEST_P(IntegerIndexTest, InitializeNewFiles) { @@ -1142,7 +1159,8 @@ TEST_P(IntegerIndexTest, InitializeNewFiles) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); ICING_ASSERT_OK(integer_index->PersistToDisk()); } @@ -1160,6 +1178,8 @@ TEST_P(IntegerIndexTest, InitializeNewFiles) { IntegerIndex::kInfoMetadataFileOffset)); EXPECT_THAT(info.magic, Eq(Info::kMagic)); EXPECT_THAT(info.last_added_document_id, Eq(kInvalidDocumentId)); + EXPECT_THAT(info.num_data_threshold_for_bucket_split, + Eq(GetParam().num_data_threshold_for_bucket_split)); // Check crcs section Crcs crcs; @@ -1183,7 +1203,8 @@ TEST_P(IntegerIndexTest, ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); // Insert some data. Index(integer_index.get(), kDefaultTestPropertyPath, /*document_id=*/0, @@ -1195,16 +1216,19 @@ TEST_P(IntegerIndexTest, // Without calling PersistToDisk, checksums will not be recomputed or synced // to disk, so initializing another instance on the same files should fail. - EXPECT_THAT(IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam()), - StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); + EXPECT_THAT( + IntegerIndex::Create(filesystem_, working_path_, + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv), + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } TEST_P(IntegerIndexTest, InitializationShouldSucceedWithPersistToDisk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index1, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); // Insert some data. Index(integer_index1.get(), kDefaultTestPropertyPath, /*document_id=*/0, @@ -1228,7 +1252,8 @@ TEST_P(IntegerIndexTest, InitializationShouldSucceedWithPersistToDisk) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index2, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); EXPECT_THAT(integer_index2->last_added_document_id(), Eq(2)); EXPECT_THAT(Query(integer_index2.get(), kDefaultTestPropertyPath, /*key_lower=*/std::numeric_limits<int64_t>::min(), @@ -1243,7 +1268,8 @@ TEST_P(IntegerIndexTest, InitializationShouldSucceedAfterDestruction) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); // Insert some data. Index(integer_index.get(), kDefaultTestPropertyPath, /*document_id=*/0, @@ -1268,7 +1294,8 @@ TEST_P(IntegerIndexTest, InitializationShouldSucceedAfterDestruction) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); EXPECT_THAT(integer_index->last_added_document_id(), Eq(2)); EXPECT_THAT(Query(integer_index.get(), kDefaultTestPropertyPath, /*key_lower=*/std::numeric_limits<int64_t>::min(), @@ -1283,7 +1310,8 @@ TEST_P(IntegerIndexTest, InitializeExistingFilesWithWrongAllCrcShouldFail) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); // Insert some data. Index(integer_index.get(), kDefaultTestPropertyPath, /*document_id=*/0, /*section_id=*/20, /*keys=*/{0, 100, -100}); @@ -1315,8 +1343,10 @@ TEST_P(IntegerIndexTest, InitializeExistingFilesWithWrongAllCrcShouldFail) { // Attempt to create the integer index with metadata containing corrupted // all_crc. This should fail. libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndex>> - integer_index_or = IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam()); + integer_index_or = + IntegerIndex::Create(filesystem_, working_path_, + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv); EXPECT_THAT(integer_index_or, StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT(integer_index_or.status().error_message(), @@ -1329,7 +1359,8 @@ TEST_P(IntegerIndexTest, InitializeExistingFilesWithCorruptedInfoShouldFail) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); // Insert some data. Index(integer_index.get(), kDefaultTestPropertyPath, /*document_id=*/0, /*section_id=*/20, /*keys=*/{0, 100, -100}); @@ -1362,8 +1393,10 @@ TEST_P(IntegerIndexTest, InitializeExistingFilesWithCorruptedInfoShouldFail) { // Attempt to create the integer index with info that doesn't match its // checksum and confirm that it fails. libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndex>> - integer_index_or = IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam()); + integer_index_or = + IntegerIndex::Create(filesystem_, working_path_, + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv); EXPECT_THAT(integer_index_or, StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT(integer_index_or.status().error_message(), @@ -1377,7 +1410,8 @@ TEST_P(IntegerIndexTest, ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); // Insert some data. Index(integer_index.get(), kDefaultTestPropertyPath, /*document_id=*/0, /*section_id=*/20, /*keys=*/{0, 100, -100}); @@ -1400,7 +1434,9 @@ TEST_P(IntegerIndexTest, std::unique_ptr<IntegerIndexStorage> storage, IntegerIndexStorage::Create( filesystem_, std::move(storage_working_path), - IntegerIndexStorage::Options(/*pre_mapping_fbv=*/GetParam()), + IntegerIndexStorage::Options( + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv), &posting_list_integer_index_serializer)); ICING_ASSERT_OK(storage->AddKeys(/*document_id=*/3, /*section_id=*/4, /*new_keys=*/{3, 4, 5})); @@ -1412,8 +1448,10 @@ TEST_P(IntegerIndexTest, // Attempt to create the integer index with corrupted storages. This should // fail. libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndex>> - integer_index_or = IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam()); + integer_index_or = + IntegerIndex::Create(filesystem_, working_path_, + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv); EXPECT_THAT(integer_index_or, StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); EXPECT_THAT(integer_index_or.status().error_message(), @@ -1421,6 +1459,41 @@ TEST_P(IntegerIndexTest, } } +TEST_P( + IntegerIndexTest, + InitializeExistingFilesWithMismatchNumDataThresholdForBucketSplitShouldFail) { + { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<IntegerIndex> integer_index, + IntegerIndex::Create(filesystem_, working_path_, + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); + // Insert some data. + Index(integer_index.get(), kDefaultTestPropertyPath, /*document_id=*/0, + /*section_id=*/20, /*keys=*/{0, 100, -100}); + Index(integer_index.get(), kDefaultTestPropertyPath, /*document_id=*/1, + /*section_id=*/2, /*keys=*/{3, -1000, 500}); + Index(integer_index.get(), kDefaultTestPropertyPath, /*document_id=*/2, + /*section_id=*/15, /*keys=*/{-6, 321, 98}); + + ICING_ASSERT_OK(integer_index->PersistToDisk()); + } + + { + // Attempt to create the integer index with different + // num_data_threshold_for_bucket_split. This should fail. + libtextclassifier3::StatusOr<std::unique_ptr<IntegerIndex>> + integer_index_or = IntegerIndex::Create( + filesystem_, working_path_, + GetParam().num_data_threshold_for_bucket_split + 1, + GetParam().pre_mapping_fbv); + EXPECT_THAT(integer_index_or, + StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); + EXPECT_THAT(integer_index_or.status().error_message(), + HasSubstr("Mismatch num_data_threshold_for_bucket_split")); + } +} + TEST_P(IntegerIndexTest, WildcardStoragePersistenceQuery) { // This test sets its schema assuming that max property storages == 32. ASSERT_THAT(IntegerIndex::kMaxPropertyStorages, Eq(32)); @@ -1586,7 +1659,8 @@ TEST_P(IntegerIndexTest, WildcardStoragePersistenceQuery) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); // Index numeric content for other properties to force our property into the // wildcard storage. @@ -1651,7 +1725,8 @@ TEST_P(IntegerIndexTest, WildcardStoragePersistenceQuery) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); EXPECT_THAT(integer_index->num_property_indices(), Eq(33)); @@ -1691,7 +1766,8 @@ TEST_P(IntegerIndexTest, ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); // Doc id = 1: insert 2 data for "prop1", "prop2" Index(integer_index.get(), kPropertyPath2, /*document_id=*/1, kSectionId2, @@ -1742,7 +1818,8 @@ TEST_P(IntegerIndexTest, ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); // Key = 1 EXPECT_THAT(Query(integer_index.get(), kPropertyPath1, /*key_lower=*/1, @@ -1968,7 +2045,8 @@ TEST_P(IntegerIndexTest, WildcardStorageWorksAfterOptimize) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); // Index numeric content for other properties to force our property into the // wildcard storage. @@ -2067,7 +2145,8 @@ TEST_P(IntegerIndexTest, WildcardStorageWorksAfterOptimize) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); EXPECT_THAT(integer_index->num_property_indices(), Eq(33)); @@ -2236,7 +2315,8 @@ TEST_P(IntegerIndexTest, WildcardStorageAvailableIndicesAfterOptimize) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); // Index numeric content for other properties to force our property into the // wildcard storage. @@ -2317,7 +2397,8 @@ TEST_P(IntegerIndexTest, WildcardStorageAvailableIndicesAfterOptimize) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<IntegerIndex> integer_index, IntegerIndex::Create(filesystem_, working_path_, - /*pre_mapping_fbv=*/GetParam())); + GetParam().num_data_threshold_for_bucket_split, + GetParam().pre_mapping_fbv)); EXPECT_THAT(integer_index->num_property_indices(), Eq(1)); @@ -2363,8 +2444,20 @@ TEST_P(IntegerIndexTest, WildcardStorageAvailableIndicesAfterOptimize) { /*document_id=*/7, expected_sections_typea)))); } -INSTANTIATE_TEST_SUITE_P(IntegerIndexTest, IntegerIndexTest, - testing::Values(true, false)); +INSTANTIATE_TEST_SUITE_P( + IntegerIndexTest, IntegerIndexTest, + testing::Values( + IntegerIndexTestParam(/*num_data_threshold_for_bucket_split_in=*/341, + /*pre_mapping_fbv_in=*/false), + IntegerIndexTestParam(/*num_data_threshold_for_bucket_split_in=*/341, + /*pre_mapping_fbv_in=*/true), + + IntegerIndexTestParam(/*num_data_threshold_for_bucket_split_in=*/16384, + /*pre_mapping_fbv_in=*/false), + IntegerIndexTestParam(/*num_data_threshold_for_bucket_split_in=*/32768, + /*pre_mapping_fbv_in=*/false), + IntegerIndexTestParam(/*num_data_threshold_for_bucket_split_in=*/65536, + /*pre_mapping_fbv_in=*/false))); } // namespace diff --git a/icing/index/numeric/numeric-index.h b/icing/index/numeric/numeric-index.h index 24b81e7..57911de 100644 --- a/icing/index/numeric/numeric-index.h +++ b/icing/index/numeric/numeric-index.h @@ -177,15 +177,17 @@ class NumericIndex : public PersistentStorage { : PersistentStorage(filesystem, std::move(working_path), working_path_type) {} - virtual libtextclassifier3::Status PersistStoragesToDisk() override = 0; + virtual libtextclassifier3::Status PersistStoragesToDisk( + bool force) override = 0; - virtual libtextclassifier3::Status PersistMetadataToDisk() override = 0; + virtual libtextclassifier3::Status PersistMetadataToDisk( + bool force) override = 0; - virtual libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum() - override = 0; + virtual libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum( + bool force) override = 0; - virtual libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum() - override = 0; + virtual libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum( + bool force) override = 0; virtual Crcs& crcs() override = 0; virtual const Crcs& crcs() const override = 0; diff --git a/icing/index/numeric/posting-list-integer-index-accessor.h b/icing/index/numeric/posting-list-integer-index-accessor.h index f0d3d25..4f667a0 100644 --- a/icing/index/numeric/posting-list-integer-index-accessor.h +++ b/icing/index/numeric/posting-list-integer-index-accessor.h @@ -100,16 +100,6 @@ class PostingListIntegerIndexAccessor : public PostingListAccessor { // posting list. libtextclassifier3::Status PrependData(const IntegerIndexData& data); - bool WantsSplit() const { - const PostingListUsed* current_pl = - preexisting_posting_list_ != nullptr - ? &preexisting_posting_list_->posting_list - : &in_memory_posting_list_; - // Only max-sized PLs get split. Smaller PLs just get copied to larger PLs. - return current_pl->size_in_bytes() == storage_->max_posting_list_bytes() && - serializer_->IsFull(current_pl); - } - private: explicit PostingListIntegerIndexAccessor( FlashIndexStorage* storage, PostingListUsed in_memory_posting_list, diff --git a/icing/index/string-section-indexing-handler.cc b/icing/index/string-section-indexing-handler.cc index 69b8889..f5e06ad 100644 --- a/icing/index/string-section-indexing-handler.cc +++ b/icing/index/string-section-indexing-handler.cc @@ -122,6 +122,17 @@ libtextclassifier3::Status StringSectionIndexingHandler::Handle( } } + // Check and sort the LiteIndex HitBuffer if we're successful. + if (status.ok() && index_.LiteIndexNeedSort()) { + std::unique_ptr<Timer> sort_timer = clock_.GetNewTimer(); + index_.SortLiteIndex(); + + if (put_document_stats != nullptr) { + put_document_stats->set_lite_index_sort_latency_ms( + sort_timer->GetElapsedMilliseconds()); + } + } + if (put_document_stats != nullptr) { put_document_stats->set_term_index_latency_ms( index_timer->GetElapsedMilliseconds()); diff --git a/icing/index/string-section-indexing-handler_test.cc b/icing/index/string-section-indexing-handler_test.cc new file mode 100644 index 0000000..2c7f5e3 --- /dev/null +++ b/icing/index/string-section-indexing-handler_test.cc @@ -0,0 +1,587 @@ +// Copyright (C) 2023 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "icing/index/string-section-indexing-handler.h" + +#include <cstdint> +#include <limits> +#include <memory> +#include <string> +#include <string_view> +#include <unordered_map> +#include <utility> +#include <vector> + +#include "icing/text_classifier/lib3/utils/base/status.h" +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "icing/document-builder.h" +#include "icing/file/filesystem.h" +#include "icing/file/portable-file-backed-proto-log.h" +#include "icing/index/hit/doc-hit-info.h" +#include "icing/index/hit/hit.h" +#include "icing/index/index.h" +#include "icing/index/iterator/doc-hit-info-iterator-test-util.h" +#include "icing/index/iterator/doc-hit-info-iterator.h" +#include "icing/legacy/index/icing-filesystem.h" +#include "icing/portable/platform.h" +#include "icing/proto/document.pb.h" +#include "icing/proto/document_wrapper.pb.h" +#include "icing/proto/schema.pb.h" +#include "icing/proto/term.pb.h" +#include "icing/schema-builder.h" +#include "icing/schema/schema-store.h" +#include "icing/schema/section.h" +#include "icing/store/document-id.h" +#include "icing/store/document-store.h" +#include "icing/testing/common-matchers.h" +#include "icing/testing/fake-clock.h" +#include "icing/testing/icu-data-file-helper.h" +#include "icing/testing/test-data.h" +#include "icing/testing/tmp-directory.h" +#include "icing/tokenization/language-segmenter-factory.h" +#include "icing/tokenization/language-segmenter.h" +#include "icing/transform/normalizer-factory.h" +#include "icing/transform/normalizer.h" +#include "icing/util/tokenized-document.h" +#include "unicode/uloc.h" + +namespace icing { +namespace lib { + +namespace { + +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::IsFalse; +using ::testing::IsTrue; +using ::testing::Test; + +// Schema type with indexable properties and section Id. +// Section Id is determined by the lexicographical order of indexable property +// path. +// Section id = 0: body +// Section id = 1: title +constexpr std::string_view kFakeType = "FakeType"; +constexpr std::string_view kPropertyBody = "body"; +constexpr std::string_view kPropertyTitle = "title"; + +constexpr SectionId kSectionIdBody = 0; +constexpr SectionId kSectionIdTitle = 1; + +// Schema type with nested indexable properties and section Id. +// Section id = 0: "name" +// Section id = 1: "nested.body" +// Section id = 3: "nested.title" +// Section id = 4: "subject" +constexpr std::string_view kNestedType = "NestedType"; +constexpr std::string_view kPropertyName = "name"; +constexpr std::string_view kPropertyNestedDoc = "nested"; +constexpr std::string_view kPropertySubject = "subject"; + +constexpr SectionId kSectionIdNestedBody = 1; + +class StringSectionIndexingHandlerTest : public Test { + protected: + void SetUp() override { + if (!IsCfStringTokenization() && !IsReverseJniTokenization()) { + ICING_ASSERT_OK( + // File generated via icu_data_file rule in //icing/BUILD. + icu_data_file_helper::SetUpICUDataFile( + GetTestFilePath("icing/icu.dat"))); + } + + base_dir_ = GetTestTempDir() + "/icing_test"; + ASSERT_THAT(filesystem_.CreateDirectoryRecursively(base_dir_.c_str()), + IsTrue()); + + index_dir_ = base_dir_ + "/index"; + schema_store_dir_ = base_dir_ + "/schema_store"; + document_store_dir_ = base_dir_ + "/document_store"; + + language_segmenter_factory::SegmenterOptions segmenter_options(ULOC_US); + ICING_ASSERT_OK_AND_ASSIGN( + lang_segmenter_, + language_segmenter_factory::Create(std::move(segmenter_options))); + + ICING_ASSERT_OK_AND_ASSIGN( + normalizer_, + normalizer_factory::Create( + /*max_term_byte_size=*/std::numeric_limits<int32_t>::max())); + + ASSERT_THAT( + filesystem_.CreateDirectoryRecursively(schema_store_dir_.c_str()), + IsTrue()); + ICING_ASSERT_OK_AND_ASSIGN( + schema_store_, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + SchemaProto schema = + SchemaBuilder() + .AddType( + SchemaTypeConfigBuilder() + .SetType(kFakeType) + .AddProperty(PropertyConfigBuilder() + .SetName(kPropertyTitle) + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName(kPropertyBody) + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType( + SchemaTypeConfigBuilder() + .SetType(kNestedType) + .AddProperty( + PropertyConfigBuilder() + .SetName(kPropertyNestedDoc) + .SetDataTypeDocument( + kFakeType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName(kPropertySubject) + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName(kPropertyName) + .SetDataTypeString(TERM_MATCH_EXACT, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + ICING_ASSERT_OK(schema_store_->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + ASSERT_TRUE( + filesystem_.CreateDirectoryRecursively(document_store_dir_.c_str())); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult doc_store_create_result, + DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, + schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); + document_store_ = std::move(doc_store_create_result.document_store); + } + + void TearDown() override { + document_store_.reset(); + schema_store_.reset(); + normalizer_.reset(); + lang_segmenter_.reset(); + + filesystem_.DeleteDirectoryRecursively(base_dir_.c_str()); + } + + Filesystem filesystem_; + IcingFilesystem icing_filesystem_; + FakeClock fake_clock_; + std::string base_dir_; + std::string index_dir_; + std::string schema_store_dir_; + std::string document_store_dir_; + + std::unique_ptr<LanguageSegmenter> lang_segmenter_; + std::unique_ptr<Normalizer> normalizer_; + std::unique_ptr<SchemaStore> schema_store_; + std::unique_ptr<DocumentStore> document_store_; +}; + +std::vector<DocHitInfo> GetHits(std::unique_ptr<DocHitInfoIterator> iterator) { + std::vector<DocHitInfo> infos; + while (iterator->Advance().ok()) { + infos.push_back(iterator->doc_hit_info()); + } + return infos; +} + +std::vector<DocHitInfoTermFrequencyPair> GetHitsWithTermFrequency( + std::unique_ptr<DocHitInfoIterator> iterator) { + std::vector<DocHitInfoTermFrequencyPair> infos; + while (iterator->Advance().ok()) { + std::vector<TermMatchInfo> matched_terms_stats; + iterator->PopulateMatchedTermsStats(&matched_terms_stats); + for (const TermMatchInfo& term_match_info : matched_terms_stats) { + infos.push_back(DocHitInfoTermFrequencyPair( + iterator->doc_hit_info(), term_match_info.term_frequencies)); + } + } + return infos; +} + +TEST_F(StringSectionIndexingHandlerTest, + HandleIntoLiteIndex_sortInIndexingNotTriggered) { + Index::Options options(index_dir_, /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<Index> index, + Index::Create(options, &filesystem_, &icing_filesystem_)); + + DocumentProto document = + DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyTitle), "foo") + .AddStringProperty(std::string(kPropertyBody), "foo bar baz") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document))); + + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id, + document_store_->Put(tokenized_document.document())); + + EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId)); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<StringSectionIndexingHandler> handler, + StringSectionIndexingHandler::Create(&fake_clock_, normalizer_.get(), + index.get())); + EXPECT_THAT( + handler->Handle(tokenized_document, document_id, /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + + EXPECT_THAT(index->last_added_document_id(), Eq(document_id)); + + // Query 'foo' + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<DocHitInfoIterator> itr, + index->GetIterator("foo", /*term_start_index=*/0, + /*unnormalized_term_length=*/0, kSectionIdMaskAll, + TermMatchType::EXACT_ONLY)); + std::vector<DocHitInfoTermFrequencyPair> hits = + GetHitsWithTermFrequency(std::move(itr)); + std::unordered_map<SectionId, Hit::TermFrequency> expected_map{ + {kSectionIdTitle, 1}, {kSectionIdBody, 1}}; + EXPECT_THAT(hits, ElementsAre(EqualsDocHitInfoWithTermFrequency( + document_id, expected_map))); + + // Query 'foo' with sectionId mask that masks all results + ICING_ASSERT_OK_AND_ASSIGN( + itr, index->GetIterator("foo", /*term_start_index=*/0, + /*unnormalized_term_length=*/0, 1U << 2, + TermMatchType::EXACT_ONLY)); + EXPECT_THAT(GetHits(std::move(itr)), IsEmpty()); +} + +TEST_F(StringSectionIndexingHandlerTest, + HandleIntoLiteIndex_sortInIndexingTriggered) { + // Create the LiteIndex with a smaller sort threshold. At 64 bytes we sort the + // HitBuffer after inserting 8 hits + Index::Options options(index_dir_, + /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/64); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<Index> index, + Index::Create(options, &filesystem_, &icing_filesystem_)); + + DocumentProto document0 = + DocumentBuilder() + .SetKey("icing", "fake_type/0") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyTitle), "foo foo foo") + .AddStringProperty(std::string(kPropertyBody), "foo bar baz") + .Build(); + DocumentProto document1 = + DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyTitle), "bar baz baz") + .AddStringProperty(std::string(kPropertyBody), "foo foo baz") + .Build(); + DocumentProto document2 = + DocumentBuilder() + .SetKey("icing", "nested_type/0") + .SetSchema(std::string(kNestedType)) + .AddDocumentProperty(std::string(kPropertyNestedDoc), document1) + .AddStringProperty(std::string(kPropertyName), "qux") + .AddStringProperty(std::string(kPropertySubject), "bar bar") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document0, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document0))); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id0, + document_store_->Put(tokenized_document0.document())); + + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document1, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document1))); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id1, + document_store_->Put(tokenized_document1.document())); + + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document2, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document2))); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id2, + document_store_->Put(tokenized_document2.document())); + EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId)); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<StringSectionIndexingHandler> handler, + StringSectionIndexingHandler::Create(&fake_clock_, normalizer_.get(), + index.get())); + + // Handle doc0 and doc1. The LiteIndex should sort and merge after adding + // these + EXPECT_THAT(handler->Handle(tokenized_document0, document_id0, + /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(handler->Handle(tokenized_document1, document_id1, + /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(index->last_added_document_id(), Eq(document_id1)); + EXPECT_THAT(index->LiteIndexNeedSort(), IsFalse()); + + // Handle doc2. The LiteIndex should have an unsorted portion after adding + EXPECT_THAT(handler->Handle(tokenized_document2, document_id2, + /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(index->last_added_document_id(), Eq(document_id2)); + + // Hits in the hit buffer: + // <term>: {(docId, sectionId, term_freq)...} + // foo: {(0, kSectionIdTitle, 3); (0, kSectionIdBody, 1); + // (1, kSectionIdBody, 2); + // (2, kSectionIdNestedBody, 2)} + // bar: {(0, kSectionIdBody, 1); + // (1, kSectionIdTitle, 1); + // (2, kSectionIdNestedTitle, 1); (2, kSectionIdSubject, 2)} + // baz: {(0, kSectionIdBody, 1); + // (1, kSectionIdTitle, 2); (1, kSectionIdBody, 1), + // (2, kSectionIdNestedTitle, 2); (2, kSectionIdNestedBody, 1)} + // qux: {(2, kSectionIdName, 1)} + + // Query 'foo' + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<DocHitInfoIterator> itr, + index->GetIterator("foo", /*term_start_index=*/0, + /*unnormalized_term_length=*/0, kSectionIdMaskAll, + TermMatchType::EXACT_ONLY)); + + // Advance the iterator and verify that we're returning hits in the correct + // order (i.e. in descending order of DocId) + ASSERT_THAT(itr->Advance(), IsOk()); + EXPECT_THAT(itr->doc_hit_info().document_id(), Eq(2)); + EXPECT_THAT(itr->doc_hit_info().hit_section_ids_mask(), + Eq(1U << kSectionIdNestedBody)); + std::vector<TermMatchInfo> matched_terms_stats; + std::unordered_map<SectionId, Hit::TermFrequency> + expected_section_ids_tf_map2 = {{kSectionIdNestedBody, 2}}; + itr->PopulateMatchedTermsStats(&matched_terms_stats); + EXPECT_THAT(matched_terms_stats, ElementsAre(EqualsTermMatchInfo( + "foo", expected_section_ids_tf_map2))); + + ASSERT_THAT(itr->Advance(), IsOk()); + EXPECT_THAT(itr->doc_hit_info().document_id(), Eq(1)); + EXPECT_THAT(itr->doc_hit_info().hit_section_ids_mask(), + Eq(1U << kSectionIdBody)); + std::unordered_map<SectionId, Hit::TermFrequency> + expected_section_ids_tf_map1 = {{kSectionIdBody, 2}}; + matched_terms_stats.clear(); + itr->PopulateMatchedTermsStats(&matched_terms_stats); + EXPECT_THAT(matched_terms_stats, ElementsAre(EqualsTermMatchInfo( + "foo", expected_section_ids_tf_map1))); + + ASSERT_THAT(itr->Advance(), IsOk()); + EXPECT_THAT(itr->doc_hit_info().document_id(), Eq(0)); + EXPECT_THAT(itr->doc_hit_info().hit_section_ids_mask(), + Eq(1U << kSectionIdTitle | 1U << kSectionIdBody)); + std::unordered_map<SectionId, Hit::TermFrequency> + expected_section_ids_tf_map0 = {{kSectionIdTitle, 3}, + {kSectionIdBody, 1}}; + matched_terms_stats.clear(); + itr->PopulateMatchedTermsStats(&matched_terms_stats); + EXPECT_THAT(matched_terms_stats, ElementsAre(EqualsTermMatchInfo( + "foo", expected_section_ids_tf_map0))); +} + +TEST_F(StringSectionIndexingHandlerTest, + HandleIntoLiteIndex_enableSortInIndexing) { + // Create the LiteIndex with a smaller sort threshold. At 64 bytes we sort the + // HitBuffer after inserting 8 hits + Index::Options options(index_dir_, + /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/false, + /*lite_index_sort_size=*/64); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<Index> index, + Index::Create(options, &filesystem_, &icing_filesystem_)); + + DocumentProto document0 = + DocumentBuilder() + .SetKey("icing", "fake_type/0") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyTitle), "foo foo foo") + .AddStringProperty(std::string(kPropertyBody), "foo bar baz") + .Build(); + DocumentProto document1 = + DocumentBuilder() + .SetKey("icing", "fake_type/1") + .SetSchema(std::string(kFakeType)) + .AddStringProperty(std::string(kPropertyTitle), "bar baz baz") + .AddStringProperty(std::string(kPropertyBody), "foo foo baz") + .Build(); + DocumentProto document2 = + DocumentBuilder() + .SetKey("icing", "nested_type/0") + .SetSchema(std::string(kNestedType)) + .AddDocumentProperty(std::string(kPropertyNestedDoc), document1) + .AddStringProperty(std::string(kPropertyName), "qux") + .AddStringProperty(std::string(kPropertySubject), "bar bar") + .Build(); + + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document0, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document0))); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id0, + document_store_->Put(tokenized_document0.document())); + + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document1, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document1))); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id1, + document_store_->Put(tokenized_document1.document())); + + ICING_ASSERT_OK_AND_ASSIGN( + TokenizedDocument tokenized_document2, + TokenizedDocument::Create(schema_store_.get(), lang_segmenter_.get(), + std::move(document2))); + ICING_ASSERT_OK_AND_ASSIGN( + DocumentId document_id2, + document_store_->Put(tokenized_document2.document())); + EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId)); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<StringSectionIndexingHandler> handler, + StringSectionIndexingHandler::Create(&fake_clock_, normalizer_.get(), + index.get())); + + // Handle all docs + EXPECT_THAT(handler->Handle(tokenized_document0, document_id0, + /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(handler->Handle(tokenized_document1, document_id1, + /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(handler->Handle(tokenized_document2, document_id2, + /*recovery_mode=*/false, + /*put_document_stats=*/nullptr), + IsOk()); + EXPECT_THAT(index->last_added_document_id(), Eq(document_id2)); + + // We've disabled sorting during indexing so the HitBuffer's unsorted section + // should exceed the sort threshold. PersistToDisk and reinitialize the + // LiteIndex with sort_at_indexing=true. + ASSERT_THAT(index->PersistToDisk(), IsOk()); + options = Index::Options(index_dir_, + /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/64); + ICING_ASSERT_OK_AND_ASSIGN( + index, Index::Create(options, &filesystem_, &icing_filesystem_)); + + // Verify that the HitBuffer has been sorted after initializing with + // sort_at_indexing enabled. + EXPECT_THAT(index->LiteIndexNeedSort(), IsFalse()); + + // Hits in the hit buffer: + // <term>: {(docId, sectionId, term_freq)...} + // foo: {(0, kSectionIdTitle, 3); (0, kSectionIdBody, 1); + // (1, kSectionIdBody, 2); + // (2, kSectionIdNestedBody, 2)} + // bar: {(0, kSectionIdBody, 1); + // (1, kSectionIdTitle, 1); + // (2, kSectionIdNestedTitle, 1); (2, kSectionIdSubject, 2)} + // baz: {(0, kSectionIdBody, 1); + // (1, kSectionIdTitle, 2); (1, kSectionIdBody, 1), + // (2, kSectionIdNestedTitle, 2); (2, kSectionIdNestedBody, 1)} + // qux: {(2, kSectionIdName, 1)} + + // Query 'foo' + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<DocHitInfoIterator> itr, + index->GetIterator("foo", /*term_start_index=*/0, + /*unnormalized_term_length=*/0, kSectionIdMaskAll, + TermMatchType::EXACT_ONLY)); + + // Advance the iterator and verify that we're returning hits in the correct + // order (i.e. in descending order of DocId) + ASSERT_THAT(itr->Advance(), IsOk()); + EXPECT_THAT(itr->doc_hit_info().document_id(), Eq(2)); + EXPECT_THAT(itr->doc_hit_info().hit_section_ids_mask(), + Eq(1U << kSectionIdNestedBody)); + std::vector<TermMatchInfo> matched_terms_stats; + std::unordered_map<SectionId, Hit::TermFrequency> + expected_section_ids_tf_map2 = {{kSectionIdNestedBody, 2}}; + itr->PopulateMatchedTermsStats(&matched_terms_stats); + EXPECT_THAT(matched_terms_stats, ElementsAre(EqualsTermMatchInfo( + "foo", expected_section_ids_tf_map2))); + + ASSERT_THAT(itr->Advance(), IsOk()); + EXPECT_THAT(itr->doc_hit_info().document_id(), Eq(1)); + EXPECT_THAT(itr->doc_hit_info().hit_section_ids_mask(), + Eq(1U << kSectionIdBody)); + std::unordered_map<SectionId, Hit::TermFrequency> + expected_section_ids_tf_map1 = {{kSectionIdBody, 2}}; + matched_terms_stats.clear(); + itr->PopulateMatchedTermsStats(&matched_terms_stats); + EXPECT_THAT(matched_terms_stats, ElementsAre(EqualsTermMatchInfo( + "foo", expected_section_ids_tf_map1))); + + ASSERT_THAT(itr->Advance(), IsOk()); + EXPECT_THAT(itr->doc_hit_info().document_id(), Eq(0)); + EXPECT_THAT(itr->doc_hit_info().hit_section_ids_mask(), + Eq(1U << kSectionIdTitle | 1U << kSectionIdBody)); + std::unordered_map<SectionId, Hit::TermFrequency> + expected_section_ids_tf_map0 = {{kSectionIdTitle, 3}, + {kSectionIdBody, 1}}; + matched_terms_stats.clear(); + itr->PopulateMatchedTermsStats(&matched_terms_stats); + EXPECT_THAT(matched_terms_stats, ElementsAre(EqualsTermMatchInfo( + "foo", expected_section_ids_tf_map0))); +} + +} // namespace + +} // namespace lib +} // namespace icing diff --git a/icing/jni.lds b/icing/jni.lds index 401682a..64fae36 100644 --- a/icing/jni.lds +++ b/icing/jni.lds @@ -1,7 +1,6 @@ VERS_1.0 { # Export JNI symbols. global: - Java_*; JNI_OnLoad; # Hide everything else diff --git a/icing/jni/icing-search-engine-jni.cc b/icing/jni/icing-search-engine-jni.cc index f2a33e0..a0883fa 100644 --- a/icing/jni/icing-search-engine-jni.cc +++ b/icing/jni/icing-search-engine-jni.cc @@ -36,10 +36,6 @@ namespace { -// JNI string constants -// Matches field name of IcingSearchEngine#nativePointer. -const char kNativePointerField[] = "nativePointer"; - bool ParseProtoFromJniByteArray(JNIEnv* env, jbyteArray bytes, google::protobuf::MessageLite* protobuf) { icing::lib::ScopedPrimitiveArrayCritical<uint8_t> scoped_array(env, bytes); @@ -61,11 +57,14 @@ jbyteArray SerializeProtoToJniByteArray(JNIEnv* env, return ret; } +struct { + jfieldID native_pointer; +} JavaIcingSearchEngineImpl; + icing::lib::IcingSearchEngine* GetIcingSearchEnginePointer(JNIEnv* env, jobject object) { - jclass cls = env->GetObjectClass(object); - jfieldID field_id = env->GetFieldID(cls, kNativePointerField, "J"); - jlong native_pointer = env->GetLongField(object, field_id); + jlong native_pointer = + env->GetLongField(object, JavaIcingSearchEngineImpl.native_pointer); return reinterpret_cast<icing::lib::IcingSearchEngine*>(native_pointer); } @@ -73,19 +72,8 @@ icing::lib::IcingSearchEngine* GetIcingSearchEnginePointer(JNIEnv* env, extern "C" { -jint JNI_OnLoad(JavaVM* vm, void* reserved) { - JNIEnv* env; - if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) { - ICING_LOG(icing::lib::ERROR) << "ERROR: GetEnv failed"; - return JNI_ERR; - } - - return JNI_VERSION_1_6; -} - -JNIEXPORT jlong JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeCreate( - JNIEnv* env, jclass clazz, jbyteArray icing_search_engine_options_bytes) { +jlong nativeCreate(JNIEnv* env, jclass clazz, + jbyteArray icing_search_engine_options_bytes) { icing::lib::IcingSearchEngineOptions options; if (!ParseProtoFromJniByteArray(env, icing_search_engine_options_bytes, &options)) { @@ -103,17 +91,13 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeCreate( return reinterpret_cast<jlong>(icing); } -JNIEXPORT void JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeDestroy( - JNIEnv* env, jclass clazz, jobject object) { +void nativeDestroy(JNIEnv* env, jclass clazz, jobject object) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); delete icing; } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeInitialize( - JNIEnv* env, jclass clazz, jobject object) { +jbyteArray nativeInitialize(JNIEnv* env, jclass clazz, jobject object) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -123,10 +107,9 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeInitialize( return SerializeProtoToJniByteArray(env, initialize_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeSetSchema( - JNIEnv* env, jclass clazz, jobject object, jbyteArray schema_bytes, - jboolean ignore_errors_and_delete_documents) { +jbyteArray nativeSetSchema(JNIEnv* env, jclass clazz, jobject object, + jbyteArray schema_bytes, + jboolean ignore_errors_and_delete_documents) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -143,9 +126,7 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeSetSchema( return SerializeProtoToJniByteArray(env, set_schema_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetSchema( - JNIEnv* env, jclass clazz, jobject object) { +jbyteArray nativeGetSchema(JNIEnv* env, jclass clazz, jobject object) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -154,9 +135,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetSchema( return SerializeProtoToJniByteArray(env, get_schema_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetSchemaType( - JNIEnv* env, jclass clazz, jobject object, jstring schema_type) { +jbyteArray nativeGetSchemaType(JNIEnv* env, jclass clazz, jobject object, + jstring schema_type) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -167,9 +147,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetSchemaType( return SerializeProtoToJniByteArray(env, get_schema_type_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativePut( - JNIEnv* env, jclass clazz, jobject object, jbyteArray document_bytes) { +jbyteArray nativePut(JNIEnv* env, jclass clazz, jobject object, + jbyteArray document_bytes) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -186,10 +165,9 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativePut( return SerializeProtoToJniByteArray(env, put_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeGet( - JNIEnv* env, jclass clazz, jobject object, jstring name_space, jstring uri, - jbyteArray result_spec_bytes) { +jbyteArray nativeGet(JNIEnv* env, jclass clazz, jobject object, + jstring name_space, jstring uri, + jbyteArray result_spec_bytes) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -208,9 +186,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeGet( return SerializeProtoToJniByteArray(env, get_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeReportUsage( - JNIEnv* env, jclass clazz, jobject object, jbyteArray usage_report_bytes) { +jbyteArray nativeReportUsage(JNIEnv* env, jclass clazz, jobject object, + jbyteArray usage_report_bytes) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -227,9 +204,7 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeReportUsage( return SerializeProtoToJniByteArray(env, report_usage_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetAllNamespaces( - JNIEnv* env, jclass clazz, jobject object) { +jbyteArray nativeGetAllNamespaces(JNIEnv* env, jclass clazz, jobject object) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -239,10 +214,9 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetAllNamespaces( return SerializeProtoToJniByteArray(env, get_all_namespaces_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetNextPage( - JNIEnv* env, jclass clazz, jobject object, jlong next_page_token, - jlong java_to_native_start_timestamp_ms) { +jbyteArray nativeGetNextPage(JNIEnv* env, jclass clazz, jobject object, + jlong next_page_token, + jlong java_to_native_start_timestamp_ms) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -263,9 +237,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetNextPage( return SerializeProtoToJniByteArray(env, next_page_result_proto); } -JNIEXPORT void JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeInvalidateNextPageToken( - JNIEnv* env, jclass clazz, jobject object, jlong next_page_token) { +void nativeInvalidateNextPageToken(JNIEnv* env, jclass clazz, jobject object, + jlong next_page_token) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -274,11 +247,11 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeInvalidateNextPageToke return; } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeSearch( - JNIEnv* env, jclass clazz, jobject object, jbyteArray search_spec_bytes, - jbyteArray scoring_spec_bytes, jbyteArray result_spec_bytes, - jlong java_to_native_start_timestamp_ms) { +jbyteArray nativeSearch(JNIEnv* env, jclass clazz, jobject object, + jbyteArray search_spec_bytes, + jbyteArray scoring_spec_bytes, + jbyteArray result_spec_bytes, + jlong java_to_native_start_timestamp_ms) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -321,10 +294,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeSearch( return SerializeProtoToJniByteArray(env, search_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeDelete( - JNIEnv* env, jclass clazz, jobject object, jstring name_space, - jstring uri) { +jbyteArray nativeDelete(JNIEnv* env, jclass clazz, jobject object, + jstring name_space, jstring uri) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -336,9 +307,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeDelete( return SerializeProtoToJniByteArray(env, delete_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeDeleteByNamespace( - JNIEnv* env, jclass clazz, jobject object, jstring name_space) { +jbyteArray nativeDeleteByNamespace(JNIEnv* env, jclass clazz, jobject object, + jstring name_space) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -349,9 +319,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeDeleteByNamespace( return SerializeProtoToJniByteArray(env, delete_by_namespace_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeDeleteBySchemaType( - JNIEnv* env, jclass clazz, jobject object, jstring schema_type) { +jbyteArray nativeDeleteBySchemaType(JNIEnv* env, jclass clazz, jobject object, + jstring schema_type) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -362,10 +331,9 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeDeleteBySchemaType( return SerializeProtoToJniByteArray(env, delete_by_schema_type_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeDeleteByQuery( - JNIEnv* env, jclass clazz, jobject object, jbyteArray search_spec_bytes, - jboolean return_deleted_document_info) { +jbyteArray nativeDeleteByQuery(JNIEnv* env, jclass clazz, jobject object, + jbyteArray search_spec_bytes, + jboolean return_deleted_document_info) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -381,9 +349,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeDeleteByQuery( return SerializeProtoToJniByteArray(env, delete_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativePersistToDisk( - JNIEnv* env, jclass clazz, jobject object, jint persist_type_code) { +jbyteArray nativePersistToDisk(JNIEnv* env, jclass clazz, jobject object, + jint persist_type_code) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -400,9 +367,7 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativePersistToDisk( return SerializeProtoToJniByteArray(env, persist_to_disk_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeOptimize( - JNIEnv* env, jclass clazz, jobject object) { +jbyteArray nativeOptimize(JNIEnv* env, jclass clazz, jobject object) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -411,9 +376,7 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeOptimize( return SerializeProtoToJniByteArray(env, optimize_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetOptimizeInfo( - JNIEnv* env, jclass clazz, jobject object) { +jbyteArray nativeGetOptimizeInfo(JNIEnv* env, jclass clazz, jobject object) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -423,9 +386,7 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetOptimizeInfo( return SerializeProtoToJniByteArray(env, get_optimize_info_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetStorageInfo( - JNIEnv* env, jclass clazz, jobject object) { +jbyteArray nativeGetStorageInfo(JNIEnv* env, jclass clazz, jobject object) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -435,9 +396,7 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetStorageInfo( return SerializeProtoToJniByteArray(env, storage_info_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeReset( - JNIEnv* env, jclass clazz, jobject object) { +jbyteArray nativeReset(JNIEnv* env, jclass clazz, jobject object) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -446,10 +405,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeReset( return SerializeProtoToJniByteArray(env, reset_result_proto); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeSearchSuggestions( - JNIEnv* env, jclass clazz, jobject object, - jbyteArray suggestion_spec_bytes) { +jbyteArray nativeSearchSuggestions(JNIEnv* env, jclass clazz, jobject object, + jbyteArray suggestion_spec_bytes) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -466,9 +423,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeSearchSuggestions( return SerializeProtoToJniByteArray(env, suggestionResponse); } -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetDebugInfo( - JNIEnv* env, jclass clazz, jobject object, jint verbosity) { +jbyteArray nativeGetDebugInfo(JNIEnv* env, jclass clazz, jobject object, + jint verbosity) { icing::lib::IcingSearchEngine* icing = GetIcingSearchEnginePointer(env, object); @@ -485,9 +441,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetDebugInfo( return SerializeProtoToJniByteArray(env, debug_info_result_proto); } -JNIEXPORT jboolean JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeShouldLog( - JNIEnv* env, jclass clazz, jshort severity, jshort verbosity) { +jboolean nativeShouldLog(JNIEnv* env, jclass clazz, jshort severity, + jshort verbosity) { if (!icing::lib::LogSeverity::Code_IsValid(severity)) { ICING_LOG(icing::lib::ERROR) << "Invalid value for logging severity: " << severity; @@ -497,9 +452,8 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeShouldLog( static_cast<icing::lib::LogSeverity::Code>(severity), verbosity); } -JNIEXPORT jboolean JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeSetLoggingLevel( - JNIEnv* env, jclass clazz, jshort severity, jshort verbosity) { +jboolean nativeSetLoggingLevel(JNIEnv* env, jclass clazz, jshort severity, + jshort verbosity) { if (!icing::lib::LogSeverity::Code_IsValid(severity)) { ICING_LOG(icing::lib::ERROR) << "Invalid value for logging severity: " << severity; @@ -509,216 +463,111 @@ Java_com_google_android_icing_IcingSearchEngineImpl_nativeSetLoggingLevel( static_cast<icing::lib::LogSeverity::Code>(severity), verbosity); } -JNIEXPORT jstring JNICALL -Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetLoggingTag( - JNIEnv* env, jclass clazz) { +jstring nativeGetLoggingTag(JNIEnv* env, jclass clazz) { return env->NewStringUTF(icing::lib::kIcingLoggingTag); } -// TODO(b/240333360) Remove the methods below for IcingSearchEngine once we have -// a sync from Jetpack to g3 to contain the refactored IcingSearchEngine(with -// IcingSearchEngineImpl). -JNIEXPORT jlong JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeCreate( - JNIEnv* env, jclass clazz, jbyteArray icing_search_engine_options_bytes) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeCreate( - env, clazz, icing_search_engine_options_bytes); -} - -JNIEXPORT void JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeDestroy(JNIEnv* env, - jclass clazz, - jobject object) { - Java_com_google_android_icing_IcingSearchEngineImpl_nativeDestroy(env, clazz, - object); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeInitialize( - JNIEnv* env, jclass clazz, jobject object) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeInitialize( - env, clazz, object); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeSetSchema( - JNIEnv* env, jclass clazz, jobject object, jbyteArray schema_bytes, - jboolean ignore_errors_and_delete_documents) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeSetSchema( - env, clazz, object, schema_bytes, ignore_errors_and_delete_documents); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeGetSchema( - JNIEnv* env, jclass clazz, jobject object) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetSchema( - env, clazz, object); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeGetSchemaType( - JNIEnv* env, jclass clazz, jobject object, jstring schema_type) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetSchemaType( - env, clazz, object, schema_type); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativePut( - JNIEnv* env, jclass clazz, jobject object, jbyteArray document_bytes) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativePut( - env, clazz, object, document_bytes); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeGet( - JNIEnv* env, jclass clazz, jobject object, jstring name_space, jstring uri, - jbyteArray result_spec_bytes) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeGet( - env, clazz, object, name_space, uri, result_spec_bytes); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeReportUsage( - JNIEnv* env, jclass clazz, jobject object, jbyteArray usage_report_bytes) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeReportUsage( - env, clazz, object, usage_report_bytes); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeGetAllNamespaces( - JNIEnv* env, jclass clazz, jobject object) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetAllNamespaces( - env, clazz, object); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeGetNextPage( - JNIEnv* env, jclass clazz, jobject object, jlong next_page_token, - jlong java_to_native_start_timestamp_ms) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetNextPage( - env, clazz, object, next_page_token, java_to_native_start_timestamp_ms); -} - -JNIEXPORT void JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeInvalidateNextPageToken( - JNIEnv* env, jclass clazz, jobject object, jlong next_page_token) { - Java_com_google_android_icing_IcingSearchEngineImpl_nativeInvalidateNextPageToken( - env, clazz, object, next_page_token); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeSearch( - JNIEnv* env, jclass clazz, jobject object, jbyteArray search_spec_bytes, - jbyteArray scoring_spec_bytes, jbyteArray result_spec_bytes, - jlong java_to_native_start_timestamp_ms) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeSearch( - env, clazz, object, search_spec_bytes, scoring_spec_bytes, - result_spec_bytes, java_to_native_start_timestamp_ms); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeDelete(JNIEnv* env, - jclass clazz, - jobject object, - jstring name_space, - jstring uri) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeDelete( - env, clazz, object, name_space, uri); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeDeleteByNamespace( - JNIEnv* env, jclass clazz, jobject object, jstring name_space) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeDeleteByNamespace( - env, clazz, object, name_space); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeDeleteBySchemaType( - JNIEnv* env, jclass clazz, jobject object, jstring schema_type) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeDeleteBySchemaType( - env, clazz, object, schema_type); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeDeleteByQuery( - JNIEnv* env, jclass clazz, jobject object, jbyteArray search_spec_bytes, - jboolean return_deleted_document_info) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeDeleteByQuery( - env, clazz, object, search_spec_bytes, return_deleted_document_info); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativePersistToDisk( - JNIEnv* env, jclass clazz, jobject object, jint persist_type_code) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativePersistToDisk( - env, clazz, object, persist_type_code); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeOptimize(JNIEnv* env, - jclass clazz, - jobject object) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeOptimize( - env, clazz, object); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeGetOptimizeInfo( - JNIEnv* env, jclass clazz, jobject object) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetOptimizeInfo( - env, clazz, object); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeGetStorageInfo( - JNIEnv* env, jclass clazz, jobject object) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetStorageInfo( - env, clazz, object); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeReset(JNIEnv* env, - jclass clazz, - jobject object) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeReset( - env, clazz, object); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeSearchSuggestions( - JNIEnv* env, jclass clazz, jobject object, - jbyteArray suggestion_spec_bytes) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeSearchSuggestions( - env, clazz, object, suggestion_spec_bytes); -} - -JNIEXPORT jbyteArray JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeGetDebugInfo( - JNIEnv* env, jclass clazz, jobject object, jint verbosity) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetDebugInfo( - env, clazz, object, verbosity); -} - -JNIEXPORT jboolean JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeShouldLog( - JNIEnv* env, jclass clazz, jshort severity, jshort verbosity) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeShouldLog( - env, clazz, severity, verbosity); -} +#pragma clang diagnostic ignored "-Wwrite-strings" +jint JNI_OnLoad(JavaVM* vm, void* reserved) { + JNIEnv* env; + if (vm->GetEnv(reinterpret_cast<void**>(&env), JNI_VERSION_1_6) != JNI_OK) { + ICING_LOG(icing::lib::ERROR) << "ERROR: GetEnv failed"; + return JNI_ERR; + } -JNIEXPORT jboolean JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeSetLoggingLevel( - JNIEnv* env, jclass clazz, jshort severity, jshort verbosity) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeSetLoggingLevel( - env, clazz, severity, verbosity); -} + // Find your class. JNI_OnLoad is called from the correct class loader context + // for this to work. + jclass java_class = + env->FindClass("com/google/android/icing/IcingSearchEngineImpl"); + if (java_class == nullptr) { + return JNI_ERR; + } + JavaIcingSearchEngineImpl.native_pointer = + env->GetFieldID(java_class, "nativePointer", "J"); + + // Register your class' native methods. + static const JNINativeMethod methods[] = { + {"nativeCreate", "([B)J", reinterpret_cast<void*>(nativeCreate)}, + {"nativeDestroy", "(Lcom/google/android/icing/IcingSearchEngineImpl;)V", + reinterpret_cast<void*>(nativeDestroy)}, + {"nativeInitialize", + "(Lcom/google/android/icing/IcingSearchEngineImpl;)[B", + reinterpret_cast<void*>(nativeInitialize)}, + {"nativeSetSchema", + "(Lcom/google/android/icing/IcingSearchEngineImpl;[BZ)[B", + reinterpret_cast<void*>(nativeSetSchema)}, + {"nativeGetSchema", + "(Lcom/google/android/icing/IcingSearchEngineImpl;)[B", + reinterpret_cast<void*>(nativeGetSchema)}, + {"nativeGetSchemaType", + "(Lcom/google/android/icing/IcingSearchEngineImpl;Ljava/lang/String;)[B", + reinterpret_cast<void*>(nativeGetSchemaType)}, + {"nativePut", "(Lcom/google/android/icing/IcingSearchEngineImpl;[B)[B", + reinterpret_cast<void*>(nativePut)}, + {"nativeGet", + "(Lcom/google/android/icing/IcingSearchEngineImpl;Ljava/lang/" + "String;Ljava/lang/String;[B)[B", + reinterpret_cast<void*>(nativeGet)}, + {"nativeReportUsage", + "(Lcom/google/android/icing/IcingSearchEngineImpl;[B)[B", + reinterpret_cast<void*>(nativeReportUsage)}, + {"nativeGetAllNamespaces", + "(Lcom/google/android/icing/IcingSearchEngineImpl;)[B", + reinterpret_cast<void*>(nativeGetAllNamespaces)}, + {"nativeGetNextPage", + "(Lcom/google/android/icing/IcingSearchEngineImpl;JJ)[B", + reinterpret_cast<void*>(nativeGetNextPage)}, + {"nativeInvalidateNextPageToken", + "(Lcom/google/android/icing/IcingSearchEngineImpl;J)V", + reinterpret_cast<void*>(nativeInvalidateNextPageToken)}, + {"nativeSearch", + "(Lcom/google/android/icing/IcingSearchEngineImpl;[B[B[BJ)[B", + reinterpret_cast<void*>(nativeSearch)}, + {"nativeDelete", + "(Lcom/google/android/icing/IcingSearchEngineImpl;Ljava/lang/" + "String;Ljava/lang/String;)[B", + reinterpret_cast<void*>(nativeDelete)}, + {"nativeDeleteByNamespace", + "(Lcom/google/android/icing/IcingSearchEngineImpl;Ljava/lang/String;)[B", + reinterpret_cast<void*>(nativeDeleteByNamespace)}, + {"nativeDeleteBySchemaType", + "(Lcom/google/android/icing/IcingSearchEngineImpl;Ljava/lang/String;)[B", + reinterpret_cast<void*>(nativeDeleteBySchemaType)}, + {"nativeDeleteByQuery", + "(Lcom/google/android/icing/IcingSearchEngineImpl;[BZ)[B", + reinterpret_cast<void*>(nativeDeleteByQuery)}, + {"nativePersistToDisk", + "(Lcom/google/android/icing/IcingSearchEngineImpl;I)[B", + reinterpret_cast<void*>(nativePersistToDisk)}, + {"nativeOptimize", "(Lcom/google/android/icing/IcingSearchEngineImpl;)[B", + reinterpret_cast<void*>(nativeOptimize)}, + {"nativeGetOptimizeInfo", + "(Lcom/google/android/icing/IcingSearchEngineImpl;)[B", + reinterpret_cast<void*>(nativeGetOptimizeInfo)}, + {"nativeGetStorageInfo", + "(Lcom/google/android/icing/IcingSearchEngineImpl;)[B", + reinterpret_cast<void*>(nativeGetStorageInfo)}, + {"nativeReset", "(Lcom/google/android/icing/IcingSearchEngineImpl;)[B", + reinterpret_cast<void*>(nativeReset)}, + {"nativeSearchSuggestions", + "(Lcom/google/android/icing/IcingSearchEngineImpl;[B)[B", + reinterpret_cast<void*>(nativeSearchSuggestions)}, + {"nativeGetDebugInfo", + "(Lcom/google/android/icing/IcingSearchEngineImpl;I)[B", + reinterpret_cast<void*>(nativeGetDebugInfo)}, + {"nativeShouldLog", "(SS)Z", reinterpret_cast<void*>(nativeShouldLog)}, + {"nativeSetLoggingLevel", "(SS)Z", + reinterpret_cast<void*>(nativeSetLoggingLevel)}, + {"nativeGetLoggingTag", "()Ljava/lang/String;", + reinterpret_cast<void*>(nativeGetLoggingTag)}, + }; + int register_natives_success = env->RegisterNatives( + java_class, methods, sizeof(methods) / sizeof(JNINativeMethod)); + if (register_natives_success != JNI_OK) { + return register_natives_success; + } -JNIEXPORT jstring JNICALL -Java_com_google_android_icing_IcingSearchEngine_nativeGetLoggingTag( - JNIEnv* env, jclass clazz) { - return Java_com_google_android_icing_IcingSearchEngineImpl_nativeGetLoggingTag( - env, clazz); + return JNI_VERSION_1_6; } } // extern "C" diff --git a/icing/join/join-children-fetcher.h b/icing/join/join-children-fetcher.h index 5f799b8..1b875bc 100644 --- a/icing/join/join-children-fetcher.h +++ b/icing/join/join-children-fetcher.h @@ -44,7 +44,7 @@ class JoinChildrenFetcher { // Get a vector of children ScoredDocumentHit by parent document id. // // TODO(b/256022027): Implement property value joins with types of string and - // int. In these cases, GetChildren should look up joinable cache to fetch + // int. In these cases, GetChildren should look up join index to fetch // joinable property value of the given parent_doc_id according to // join_spec_.parent_property_expression, and then fetch children by the // corresponding map in this class using the joinable property value. diff --git a/icing/join/join-processor.h b/icing/join/join-processor.h index 347ce85..517e9db 100644 --- a/icing/join/join-processor.h +++ b/icing/join/join-processor.h @@ -22,7 +22,7 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/join/join-children-fetcher.h" -#include "icing/join/qualified-id-type-joinable-index.h" +#include "icing/join/qualified-id-join-index.h" #include "icing/proto/search.pb.h" #include "icing/schema/schema-store.h" #include "icing/scoring/scored-document-hit.h" @@ -35,10 +35,10 @@ class JoinProcessor { public: static constexpr std::string_view kQualifiedIdExpr = "this.qualifiedId()"; - explicit JoinProcessor( - const DocumentStore* doc_store, const SchemaStore* schema_store, - const QualifiedIdTypeJoinableIndex* qualified_id_join_index, - int64_t current_time_ms) + explicit JoinProcessor(const DocumentStore* doc_store, + const SchemaStore* schema_store, + const QualifiedIdJoinIndex* qualified_id_join_index, + int64_t current_time_ms) : doc_store_(doc_store), schema_store_(schema_store), qualified_id_join_index_(qualified_id_join_index), @@ -72,14 +72,13 @@ class JoinProcessor { // - kInvalidDocumentId if the given document is not found, doesn't have // qualified id joinable type for the given property_path, or doesn't have // joinable value (an optional property) - // - Any other QualifiedIdTypeJoinableIndex errors + // - Any other QualifiedIdJoinIndex errors libtextclassifier3::StatusOr<DocumentId> FetchReferencedQualifiedId( const DocumentId& document_id, const std::string& property_path) const; const DocumentStore* doc_store_; // Does not own. const SchemaStore* schema_store_; // Does not own. - const QualifiedIdTypeJoinableIndex* - qualified_id_join_index_; // Does not own. + const QualifiedIdJoinIndex* qualified_id_join_index_; // Does not own. int64_t current_time_ms_; }; diff --git a/icing/join/join-processor_test.cc b/icing/join/join-processor_test.cc index 95d1392..f503442 100644 --- a/icing/join/join-processor_test.cc +++ b/icing/join/join-processor_test.cc @@ -25,8 +25,8 @@ #include "icing/document-builder.h" #include "icing/file/filesystem.h" #include "icing/file/portable-file-backed-proto-log.h" +#include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id-join-indexing-handler.h" -#include "icing/join/qualified-id-type-joinable-index.h" #include "icing/portable/platform.h" #include "icing/proto/document.pb.h" #include "icing/proto/document_wrapper.pb.h" @@ -118,20 +118,21 @@ class JoinProcessorTest : public ::testing::Test { IsTrue()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, doc_store_dir_, &fake_clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); doc_store_ = std::move(create_result.document_store); ICING_ASSERT_OK_AND_ASSIGN( qualified_id_join_index_, - QualifiedIdTypeJoinableIndex::Create( - filesystem_, qualified_id_join_index_dir_, - /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/false)); + QualifiedIdJoinIndex::Create(filesystem_, qualified_id_join_index_dir_, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false)); } void TearDown() override { @@ -185,7 +186,7 @@ class JoinProcessorTest : public ::testing::Test { std::unique_ptr<LanguageSegmenter> lang_segmenter_; std::unique_ptr<SchemaStore> schema_store_; std::unique_ptr<DocumentStore> doc_store_; - std::unique_ptr<QualifiedIdTypeJoinableIndex> qualified_id_join_index_; + std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index_; FakeClock fake_clock_; }; diff --git a/icing/join/qualified-id-type-joinable-index.cc b/icing/join/qualified-id-join-index.cc index a1df3d0..07b5627 100644 --- a/icing/join/qualified-id-type-joinable-index.cc +++ b/icing/join/qualified-id-join-index.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "icing/join/qualified-id-type-joinable-index.h" +#include "icing/join/qualified-id-join-index.h" #include <cstring> #include <memory> @@ -43,6 +43,11 @@ namespace lib { namespace { +// Set 1M for max # of qualified id entries and 10 bytes for key-value bytes. +// This will take at most 23 MiB disk space and mmap for persistent hash map. +static constexpr int32_t kDocJoinInfoMapperMaxNumEntries = 1 << 20; +static constexpr int32_t kDocJoinInfoMapperAverageKVByteSize = 10; + static constexpr int32_t kDocJoinInfoMapperDynamicTrieMaxSize = 128 * 1024 * 1024; // 128 MiB @@ -69,12 +74,10 @@ std::string GetQualifiedIdStoragePath(std::string_view working_path) { } // namespace -/* static */ libtextclassifier3::StatusOr< - std::unique_ptr<QualifiedIdTypeJoinableIndex>> -QualifiedIdTypeJoinableIndex::Create(const Filesystem& filesystem, - std::string working_path, - bool pre_mapping_fbv, - bool use_persistent_hash_map) { +/* static */ libtextclassifier3::StatusOr<std::unique_ptr<QualifiedIdJoinIndex>> +QualifiedIdJoinIndex::Create(const Filesystem& filesystem, + std::string working_path, bool pre_mapping_fbv, + bool use_persistent_hash_map) { if (!filesystem.FileExists(GetMetadataFilePath(working_path).c_str()) || !filesystem.DirectoryExists( GetDocJoinInfoMapperPath(working_path).c_str()) || @@ -90,7 +93,7 @@ QualifiedIdTypeJoinableIndex::Create(const Filesystem& filesystem, pre_mapping_fbv, use_persistent_hash_map); } -QualifiedIdTypeJoinableIndex::~QualifiedIdTypeJoinableIndex() { +QualifiedIdJoinIndex::~QualifiedIdJoinIndex() { if (!PersistToDisk().ok()) { ICING_LOG(WARNING) << "Failed to persist qualified id type joinable index " "to disk while destructing " @@ -98,8 +101,10 @@ QualifiedIdTypeJoinableIndex::~QualifiedIdTypeJoinableIndex() { } } -libtextclassifier3::Status QualifiedIdTypeJoinableIndex::Put( +libtextclassifier3::Status QualifiedIdJoinIndex::Put( const DocJoinInfo& doc_join_info, std::string_view ref_qualified_id_str) { + SetDirty(); + if (!doc_join_info.is_valid()) { return absl_ports::InvalidArgumentError( "Cannot put data for an invalid DocJoinInfo"); @@ -123,8 +128,8 @@ libtextclassifier3::Status QualifiedIdTypeJoinableIndex::Put( return libtextclassifier3::Status::OK; } -libtextclassifier3::StatusOr<std::string_view> -QualifiedIdTypeJoinableIndex::Get(const DocJoinInfo& doc_join_info) const { +libtextclassifier3::StatusOr<std::string_view> QualifiedIdJoinIndex::Get( + const DocJoinInfo& doc_join_info) const { if (!doc_join_info.is_valid()) { return absl_ports::InvalidArgumentError( "Cannot get data for an invalid DocJoinInfo"); @@ -139,7 +144,7 @@ QualifiedIdTypeJoinableIndex::Get(const DocJoinInfo& doc_join_info) const { return std::string_view(data, strlen(data)); } -libtextclassifier3::Status QualifiedIdTypeJoinableIndex::Optimize( +libtextclassifier3::Status QualifiedIdJoinIndex::Optimize( const std::vector<DocumentId>& document_id_old_to_new, DocumentId new_last_added_document_id) { std::string temp_working_path = working_path_ + "_temp"; @@ -157,10 +162,9 @@ libtextclassifier3::Status QualifiedIdTypeJoinableIndex::Optimize( // Transfer all data from the current to new qualified id type joinable // index. Also PersistToDisk and destruct the instance after finishing, so // we can safely swap directories later. - ICING_ASSIGN_OR_RETURN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> new_index, - Create(filesystem_, temp_working_path_ddir.dir(), pre_mapping_fbv_, - use_persistent_hash_map_)); + ICING_ASSIGN_OR_RETURN(std::unique_ptr<QualifiedIdJoinIndex> new_index, + Create(filesystem_, temp_working_path_ddir.dir(), + pre_mapping_fbv_, use_persistent_hash_map_)); ICING_RETURN_IF_ERROR( TransferIndex(document_id_old_to_new, new_index.get())); new_index->set_last_added_document_id(new_last_added_document_id); @@ -190,7 +194,9 @@ libtextclassifier3::Status QualifiedIdTypeJoinableIndex::Optimize( doc_join_info_mapper_, PersistentHashMapKeyMapper<int32_t>::Create( filesystem_, GetDocJoinInfoMapperPath(working_path_), - pre_mapping_fbv_)); + pre_mapping_fbv_, + /*max_num_entries=*/kDocJoinInfoMapperMaxNumEntries, + /*average_kv_byte_size=*/kDocJoinInfoMapperAverageKVByteSize)); } else { ICING_ASSIGN_OR_RETURN( doc_join_info_mapper_, @@ -210,7 +216,9 @@ libtextclassifier3::Status QualifiedIdTypeJoinableIndex::Optimize( return libtextclassifier3::Status::OK; } -libtextclassifier3::Status QualifiedIdTypeJoinableIndex::Clear() { +libtextclassifier3::Status QualifiedIdJoinIndex::Clear() { + SetDirty(); + doc_join_info_mapper_.reset(); // Discard and reinitialize doc join info mapper. std::string doc_join_info_mapper_path = @@ -221,8 +229,9 @@ libtextclassifier3::Status QualifiedIdTypeJoinableIndex::Clear() { ICING_ASSIGN_OR_RETURN( doc_join_info_mapper_, PersistentHashMapKeyMapper<int32_t>::Create( - filesystem_, std::move(doc_join_info_mapper_path), - pre_mapping_fbv_)); + filesystem_, std::move(doc_join_info_mapper_path), pre_mapping_fbv_, + /*max_num_entries=*/kDocJoinInfoMapperMaxNumEntries, + /*average_kv_byte_size=*/kDocJoinInfoMapperAverageKVByteSize)); } else { ICING_RETURN_IF_ERROR(DynamicTrieKeyMapper<int32_t>::Delete( filesystem_, doc_join_info_mapper_path)); @@ -243,12 +252,11 @@ libtextclassifier3::Status QualifiedIdTypeJoinableIndex::Clear() { return libtextclassifier3::Status::OK; } -/* static */ libtextclassifier3::StatusOr< - std::unique_ptr<QualifiedIdTypeJoinableIndex>> -QualifiedIdTypeJoinableIndex::InitializeNewFiles(const Filesystem& filesystem, - std::string&& working_path, - bool pre_mapping_fbv, - bool use_persistent_hash_map) { +/* static */ libtextclassifier3::StatusOr<std::unique_ptr<QualifiedIdJoinIndex>> +QualifiedIdJoinIndex::InitializeNewFiles(const Filesystem& filesystem, + std::string&& working_path, + bool pre_mapping_fbv, + bool use_persistent_hash_map) { // Create working directory. if (!filesystem.CreateDirectoryRecursively(working_path.c_str())) { return absl_ports::InternalError( @@ -262,8 +270,9 @@ QualifiedIdTypeJoinableIndex::InitializeNewFiles(const Filesystem& filesystem, ICING_ASSIGN_OR_RETURN( doc_join_info_mapper, PersistentHashMapKeyMapper<int32_t>::Create( - filesystem, GetDocJoinInfoMapperPath(working_path), - pre_mapping_fbv)); + filesystem, GetDocJoinInfoMapperPath(working_path), pre_mapping_fbv, + /*max_num_entries=*/kDocJoinInfoMapperMaxNumEntries, + /*average_kv_byte_size=*/kDocJoinInfoMapperAverageKVByteSize)); } else { ICING_ASSIGN_OR_RETURN( doc_join_info_mapper, @@ -282,8 +291,8 @@ QualifiedIdTypeJoinableIndex::InitializeNewFiles(const Filesystem& filesystem, /*pre_mapping_mmap_size=*/pre_mapping_fbv ? 1024 * 1024 : 0)); // Create instance. - auto new_index = std::unique_ptr<QualifiedIdTypeJoinableIndex>( - new QualifiedIdTypeJoinableIndex( + auto new_index = + std::unique_ptr<QualifiedIdJoinIndex>(new QualifiedIdJoinIndex( filesystem, std::move(working_path), /*metadata_buffer=*/std::make_unique<uint8_t[]>(kMetadataFileSize), std::move(doc_join_info_mapper), std::move(qualified_id_storage), @@ -298,11 +307,11 @@ QualifiedIdTypeJoinableIndex::InitializeNewFiles(const Filesystem& filesystem, return new_index; } -/* static */ libtextclassifier3::StatusOr< - std::unique_ptr<QualifiedIdTypeJoinableIndex>> -QualifiedIdTypeJoinableIndex::InitializeExistingFiles( - const Filesystem& filesystem, std::string&& working_path, - bool pre_mapping_fbv, bool use_persistent_hash_map) { +/* static */ libtextclassifier3::StatusOr<std::unique_ptr<QualifiedIdJoinIndex>> +QualifiedIdJoinIndex::InitializeExistingFiles(const Filesystem& filesystem, + std::string&& working_path, + bool pre_mapping_fbv, + bool use_persistent_hash_map) { // PRead metadata file. auto metadata_buffer = std::make_unique<uint8_t[]>(kMetadataFileSize); if (!filesystem.PRead(GetMetadataFilePath(working_path).c_str(), @@ -328,8 +337,9 @@ QualifiedIdTypeJoinableIndex::InitializeExistingFiles( ICING_ASSIGN_OR_RETURN( doc_join_info_mapper, PersistentHashMapKeyMapper<int32_t>::Create( - filesystem, GetDocJoinInfoMapperPath(working_path), - pre_mapping_fbv)); + filesystem, GetDocJoinInfoMapperPath(working_path), pre_mapping_fbv, + /*max_num_entries=*/kDocJoinInfoMapperMaxNumEntries, + /*average_kv_byte_size=*/kDocJoinInfoMapperAverageKVByteSize)); } else { ICING_ASSIGN_OR_RETURN( doc_join_info_mapper, @@ -348,8 +358,8 @@ QualifiedIdTypeJoinableIndex::InitializeExistingFiles( /*pre_mapping_mmap_size=*/pre_mapping_fbv ? 1024 * 1024 : 0)); // Create instance. - auto type_joinable_index = std::unique_ptr<QualifiedIdTypeJoinableIndex>( - new QualifiedIdTypeJoinableIndex( + auto type_joinable_index = + std::unique_ptr<QualifiedIdJoinIndex>(new QualifiedIdJoinIndex( filesystem, std::move(working_path), std::move(metadata_buffer), std::move(doc_join_info_mapper), std::move(qualified_id_storage), pre_mapping_fbv, use_persistent_hash_map)); @@ -364,9 +374,9 @@ QualifiedIdTypeJoinableIndex::InitializeExistingFiles( return type_joinable_index; } -libtextclassifier3::Status QualifiedIdTypeJoinableIndex::TransferIndex( +libtextclassifier3::Status QualifiedIdJoinIndex::TransferIndex( const std::vector<DocumentId>& document_id_old_to_new, - QualifiedIdTypeJoinableIndex* new_index) const { + QualifiedIdJoinIndex* new_index) const { std::unique_ptr<KeyMapper<int32_t>::Iterator> iter = doc_join_info_mapper_->GetIterator(); while (iter->Advance()) { @@ -394,8 +404,12 @@ libtextclassifier3::Status QualifiedIdTypeJoinableIndex::TransferIndex( return libtextclassifier3::Status::OK; } -libtextclassifier3::Status -QualifiedIdTypeJoinableIndex::PersistMetadataToDisk() { +libtextclassifier3::Status QualifiedIdJoinIndex::PersistMetadataToDisk( + bool force) { + if (!force && !is_info_dirty() && !is_storage_dirty()) { + return libtextclassifier3::Status::OK; + } + std::string metadata_file_path = GetMetadataFilePath(working_path_); ScopedFd sfd(filesystem_.OpenForWrite(metadata_file_path.c_str())); @@ -415,20 +429,32 @@ QualifiedIdTypeJoinableIndex::PersistMetadataToDisk() { return libtextclassifier3::Status::OK; } -libtextclassifier3::Status -QualifiedIdTypeJoinableIndex::PersistStoragesToDisk() { +libtextclassifier3::Status QualifiedIdJoinIndex::PersistStoragesToDisk( + bool force) { + if (!force && !is_storage_dirty()) { + return libtextclassifier3::Status::OK; + } + ICING_RETURN_IF_ERROR(doc_join_info_mapper_->PersistToDisk()); ICING_RETURN_IF_ERROR(qualified_id_storage_->PersistToDisk()); return libtextclassifier3::Status::OK; } -libtextclassifier3::StatusOr<Crc32> -QualifiedIdTypeJoinableIndex::ComputeInfoChecksum() { +libtextclassifier3::StatusOr<Crc32> QualifiedIdJoinIndex::ComputeInfoChecksum( + bool force) { + if (!force && !is_info_dirty()) { + return Crc32(crcs().component_crcs.info_crc); + } + return info().ComputeChecksum(); } libtextclassifier3::StatusOr<Crc32> -QualifiedIdTypeJoinableIndex::ComputeStoragesChecksum() { +QualifiedIdJoinIndex::ComputeStoragesChecksum(bool force) { + if (!force && !is_storage_dirty()) { + return Crc32(crcs().component_crcs.storages_crc); + } + ICING_ASSIGN_OR_RETURN(Crc32 doc_join_info_mapper_crc, doc_join_info_mapper_->ComputeChecksum()); ICING_ASSIGN_OR_RETURN(Crc32 qualified_id_storage_crc, diff --git a/icing/join/qualified-id-type-joinable-index.h b/icing/join/qualified-id-join-index.h index 4844433..86297cd 100644 --- a/icing/join/qualified-id-type-joinable-index.h +++ b/icing/join/qualified-id-join-index.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef ICING_JOIN_QUALIFIED_ID_TYPE_JOINABLE_INDEX_H_ -#define ICING_JOIN_QUALIFIED_ID_TYPE_JOINABLE_INDEX_H_ +#ifndef ICING_JOIN_QUALIFIED_ID_JOIN_INDEX_H_ +#define ICING_JOIN_QUALIFIED_ID_JOIN_INDEX_H_ #include <cstdint> #include <memory> @@ -34,9 +34,9 @@ namespace icing { namespace lib { -// QualifiedIdTypeJoinableIndex: a class to maintain data mapping DocJoinInfo to +// QualifiedIdJoinIndex: a class to maintain data mapping DocJoinInfo to // joinable qualified ids and delete propagation info. -class QualifiedIdTypeJoinableIndex : public PersistentStorage { +class QualifiedIdJoinIndex : public PersistentStorage { public: struct Info { static constexpr int32_t kMagic = 0x48cabdc6; @@ -61,23 +61,23 @@ class QualifiedIdTypeJoinableIndex : public PersistentStorage { static constexpr WorkingPathType kWorkingPathType = WorkingPathType::kDirectory; - // Creates a QualifiedIdTypeJoinableIndex instance to store qualified ids for - // future joining search. If any of the underlying file is missing, then - // delete the whole working_path and (re)initialize with new ones. Otherwise - // initialize and create the instance by existing files. + // Creates a QualifiedIdJoinIndex instance to store qualified ids for future + // joining search. If any of the underlying file is missing, then delete the + // whole working_path and (re)initialize with new ones. Otherwise initialize + // and create the instance by existing files. // // filesystem: Object to make system level calls // working_path: Specifies the working path for PersistentStorage. - // QualifiedIdTypeJoinableIndex uses working path as working - // directory and all related files will be stored under this - // directory. It takes full ownership and of working_path_, - // including creation/deletion. It is the caller's - // responsibility to specify correct working path and avoid - // mixing different persistent storages together under the same - // path. Also the caller has the ownership for the parent - // directory of working_path_, and it is responsible for parent - // directory creation/deletion. See PersistentStorage for more - // details about the concept of working_path. + // QualifiedIdJoinIndex uses working path as working directory + // and all related files will be stored under this directory. It + // takes full ownership and of working_path_, including + // creation/deletion. It is the caller's responsibility to + // specify correct working path and avoid mixing different + // persistent storages together under the same path. Also the + // caller has the ownership for the parent directory of + // working_path_, and it is responsible for parent directory + // creation/deletion. See PersistentStorage for more details + // about the concept of working_path. // pre_mapping_fbv: flag indicating whether memory map max possible file size // for underlying FileBackedVector before growing the actual // file size. @@ -90,12 +90,11 @@ class QualifiedIdTypeJoinableIndex : public PersistentStorage { // checksum // - INTERNAL_ERROR on I/O errors // - Any KeyMapper errors - static libtextclassifier3::StatusOr< - std::unique_ptr<QualifiedIdTypeJoinableIndex>> + static libtextclassifier3::StatusOr<std::unique_ptr<QualifiedIdJoinIndex>> Create(const Filesystem& filesystem, std::string working_path, bool pre_mapping_fbv, bool use_persistent_hash_map); - // Deletes QualifiedIdTypeJoinableIndex under working_path. + // Deletes QualifiedIdJoinIndex under working_path. // // Returns: // - OK on success @@ -107,15 +106,13 @@ class QualifiedIdTypeJoinableIndex : public PersistentStorage { } // Delete copy and move constructor/assignment operator. - QualifiedIdTypeJoinableIndex(const QualifiedIdTypeJoinableIndex&) = delete; - QualifiedIdTypeJoinableIndex& operator=(const QualifiedIdTypeJoinableIndex&) = - delete; + QualifiedIdJoinIndex(const QualifiedIdJoinIndex&) = delete; + QualifiedIdJoinIndex& operator=(const QualifiedIdJoinIndex&) = delete; - QualifiedIdTypeJoinableIndex(QualifiedIdTypeJoinableIndex&&) = delete; - QualifiedIdTypeJoinableIndex& operator=(QualifiedIdTypeJoinableIndex&&) = - delete; + QualifiedIdJoinIndex(QualifiedIdJoinIndex&&) = delete; + QualifiedIdJoinIndex& operator=(QualifiedIdJoinIndex&&) = delete; - ~QualifiedIdTypeJoinableIndex() override; + ~QualifiedIdJoinIndex() override; // Puts a new data into index: DocJoinInfo (DocumentId, JoinablePropertyId) // references to ref_qualified_id_str (the identifier of another document). @@ -175,6 +172,8 @@ class QualifiedIdTypeJoinableIndex : public PersistentStorage { } void set_last_added_document_id(DocumentId document_id) { + SetInfoDirty(); + Info& info_ref = info(); if (info_ref.last_added_document_id == kInvalidDocumentId || document_id > info_ref.last_added_document_id) { @@ -183,7 +182,7 @@ class QualifiedIdTypeJoinableIndex : public PersistentStorage { } private: - explicit QualifiedIdTypeJoinableIndex( + explicit QualifiedIdJoinIndex( const Filesystem& filesystem, std::string&& working_path, std::unique_ptr<uint8_t[]> metadata_buffer, std::unique_ptr<KeyMapper<int32_t>> doc_join_info_mapper, @@ -195,15 +194,15 @@ class QualifiedIdTypeJoinableIndex : public PersistentStorage { doc_join_info_mapper_(std::move(doc_join_info_mapper)), qualified_id_storage_(std::move(qualified_id_storage)), pre_mapping_fbv_(pre_mapping_fbv), - use_persistent_hash_map_(use_persistent_hash_map) {} + use_persistent_hash_map_(use_persistent_hash_map), + is_info_dirty_(false), + is_storage_dirty_(false) {} - static libtextclassifier3::StatusOr< - std::unique_ptr<QualifiedIdTypeJoinableIndex>> + static libtextclassifier3::StatusOr<std::unique_ptr<QualifiedIdJoinIndex>> InitializeNewFiles(const Filesystem& filesystem, std::string&& working_path, bool pre_mapping_fbv, bool use_persistent_hash_map); - static libtextclassifier3::StatusOr< - std::unique_ptr<QualifiedIdTypeJoinableIndex>> + static libtextclassifier3::StatusOr<std::unique_ptr<QualifiedIdJoinIndex>> InitializeExistingFiles(const Filesystem& filesystem, std::string&& working_path, bool pre_mapping_fbv, bool use_persistent_hash_map); @@ -217,34 +216,35 @@ class QualifiedIdTypeJoinableIndex : public PersistentStorage { // - INTERNAL_ERROR on I/O error libtextclassifier3::Status TransferIndex( const std::vector<DocumentId>& document_id_old_to_new, - QualifiedIdTypeJoinableIndex* new_index) const; + QualifiedIdJoinIndex* new_index) const; // Flushes contents of metadata file. // // Returns: // - OK on success // - INTERNAL_ERROR on I/O error - libtextclassifier3::Status PersistMetadataToDisk() override; + libtextclassifier3::Status PersistMetadataToDisk(bool force) override; // Flushes contents of all storages to underlying files. // // Returns: // - OK on success // - INTERNAL_ERROR on I/O error - libtextclassifier3::Status PersistStoragesToDisk() override; + libtextclassifier3::Status PersistStoragesToDisk(bool force) override; // Computes and returns Info checksum. // // Returns: // - Crc of the Info on success - libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum() override; + libtextclassifier3::StatusOr<Crc32> ComputeInfoChecksum(bool force) override; // Computes and returns all storages checksum. // // Returns: // - Crc of all storages on success // - INTERNAL_ERROR if any data inconsistency - libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum() override; + libtextclassifier3::StatusOr<Crc32> ComputeStoragesChecksum( + bool force) override; Crcs& crcs() override { return *reinterpret_cast<Crcs*>(metadata_buffer_.get() + @@ -266,6 +266,17 @@ class QualifiedIdTypeJoinableIndex : public PersistentStorage { kInfoMetadataBufferOffset); } + void SetInfoDirty() { is_info_dirty_ = true; } + // When storage is dirty, we have to set info dirty as well. So just expose + // SetDirty to set both. + void SetDirty() { + is_info_dirty_ = true; + is_storage_dirty_ = true; + } + + bool is_info_dirty() const { return is_info_dirty_; } + bool is_storage_dirty() const { return is_storage_dirty_; } + // Metadata buffer std::unique_ptr<uint8_t[]> metadata_buffer_; @@ -286,9 +297,12 @@ class QualifiedIdTypeJoinableIndex : public PersistentStorage { // Flag indicating whether use persistent hash map as the key mapper (if // false, then fall back to dynamic trie key mapper). bool use_persistent_hash_map_; + + bool is_info_dirty_; + bool is_storage_dirty_; }; } // namespace lib } // namespace icing -#endif // ICING_JOIN_QUALIFIED_ID_TYPE_JOINABLE_INDEX_H_ +#endif // ICING_JOIN_QUALIFIED_ID_JOIN_INDEX_H_ diff --git a/icing/join/qualified-id-type-joinable-index_test.cc b/icing/join/qualified-id-join-index_test.cc index 8ef9167..3d59f4b 100644 --- a/icing/join/qualified-id-type-joinable-index_test.cc +++ b/icing/join/qualified-id-join-index_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "icing/join/qualified-id-type-joinable-index.h" +#include "icing/join/qualified-id-join-index.h" #include <memory> #include <string> @@ -49,7 +49,7 @@ using ::testing::Pointee; using ::testing::SizeIs; using Crcs = PersistentStorage::Crcs; -using Info = QualifiedIdTypeJoinableIndex::Info; +using Info = QualifiedIdJoinIndex::Info; static constexpr int32_t kCorruptedValueOffset = 3; @@ -63,7 +63,7 @@ struct QualifiedIdJoinIndexTestParam { use_persistent_hash_map(use_persistent_hash_map_in) {} }; -class QualifiedIdTypeJoinableIndexTest +class QualifiedIdJoinIndexTest : public ::testing::TestWithParam<QualifiedIdJoinIndexTestParam> { protected: void SetUp() override { @@ -71,7 +71,7 @@ class QualifiedIdTypeJoinableIndexTest ASSERT_THAT(filesystem_.CreateDirectoryRecursively(base_dir_.c_str()), IsTrue()); - working_path_ = base_dir_ + "/qualified_id_type_joinable_index_test"; + working_path_ = base_dir_ + "/qualified_id_join_index_test"; } void TearDown() override { @@ -83,27 +83,26 @@ class QualifiedIdTypeJoinableIndexTest std::string working_path_; }; -TEST_P(QualifiedIdTypeJoinableIndexTest, InvalidWorkingPath) { +TEST_P(QualifiedIdJoinIndexTest, InvalidWorkingPath) { const QualifiedIdJoinIndexTestParam& param = GetParam(); - EXPECT_THAT( - QualifiedIdTypeJoinableIndex::Create( - filesystem_, "/dev/null/qualified_id_type_joinable_index_test", - param.pre_mapping_fbv, param.use_persistent_hash_map), - StatusIs(libtextclassifier3::StatusCode::INTERNAL)); + EXPECT_THAT(QualifiedIdJoinIndex::Create( + filesystem_, "/dev/null/qualified_id_join_index_test", + param.pre_mapping_fbv, param.use_persistent_hash_map), + StatusIs(libtextclassifier3::StatusCode::INTERNAL)); } -TEST_P(QualifiedIdTypeJoinableIndexTest, InitializeNewFiles) { +TEST_P(QualifiedIdJoinIndexTest, InitializeNewFiles) { const QualifiedIdJoinIndexTestParam& param = GetParam(); { - // Create new qualified id type joinable index + // Create new qualified id join index ASSERT_FALSE(filesystem_.DirectoryExists(working_path_.c_str())); ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); EXPECT_THAT(index, Pointee(IsEmpty())); ICING_ASSERT_OK(index->PersistToDisk()); @@ -113,25 +112,23 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, InitializeNewFiles) { // sections. const std::string metadata_file_path = absl_ports::StrCat(working_path_, "/metadata"); - auto metadata_buffer = std::make_unique<uint8_t[]>( - QualifiedIdTypeJoinableIndex::kMetadataFileSize); + auto metadata_buffer = + std::make_unique<uint8_t[]>(QualifiedIdJoinIndex::kMetadataFileSize); ASSERT_THAT( filesystem_.PRead(metadata_file_path.c_str(), metadata_buffer.get(), - QualifiedIdTypeJoinableIndex::kMetadataFileSize, + QualifiedIdJoinIndex::kMetadataFileSize, /*offset=*/0), IsTrue()); // Check info section const Info* info = reinterpret_cast<const Info*>( - metadata_buffer.get() + - QualifiedIdTypeJoinableIndex::kInfoMetadataBufferOffset); + metadata_buffer.get() + QualifiedIdJoinIndex::kInfoMetadataBufferOffset); EXPECT_THAT(info->magic, Eq(Info::kMagic)); EXPECT_THAT(info->last_added_document_id, Eq(kInvalidDocumentId)); // Check crcs section const Crcs* crcs = reinterpret_cast<const Crcs*>( - metadata_buffer.get() + - QualifiedIdTypeJoinableIndex::kCrcsMetadataBufferOffset); + metadata_buffer.get() + QualifiedIdJoinIndex::kCrcsMetadataBufferOffset); // There are some initial info in KeyMapper, so storages_crc should be // non-zero. EXPECT_THAT(crcs->component_crcs.storages_crc, Ne(0)); @@ -146,16 +143,16 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, InitializeNewFiles) { .Get())); } -TEST_P(QualifiedIdTypeJoinableIndexTest, +TEST_P(QualifiedIdJoinIndexTest, InitializationShouldFailWithoutPersistToDiskOrDestruction) { const QualifiedIdJoinIndexTestParam& param = GetParam(); - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); // Insert some data. ICING_ASSERT_OK( @@ -171,24 +168,23 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, // Without calling PersistToDisk, checksums will not be recomputed or synced // to disk, so initializing another instance on the same files should fail. - EXPECT_THAT(QualifiedIdTypeJoinableIndex::Create( - filesystem_, working_path_, param.pre_mapping_fbv, - param.use_persistent_hash_map), + EXPECT_THAT(QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map), StatusIs(param.use_persistent_hash_map ? libtextclassifier3::StatusCode::FAILED_PRECONDITION : libtextclassifier3::StatusCode::INTERNAL)); } -TEST_P(QualifiedIdTypeJoinableIndexTest, - InitializationShouldSucceedWithPersistToDisk) { +TEST_P(QualifiedIdJoinIndexTest, InitializationShouldSucceedWithPersistToDisk) { const QualifiedIdJoinIndexTestParam& param = GetParam(); - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index1, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index1, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); // Insert some data. ICING_ASSERT_OK( @@ -208,10 +204,10 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, ICING_EXPECT_OK(index1->PersistToDisk()); ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index2, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index2, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); EXPECT_THAT(index2, Pointee(SizeIs(3))); EXPECT_THAT( index2->Get(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20)), @@ -224,17 +220,16 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, IsOkAndHolds(/*ref_qualified_id_str=*/"namespace#uriC")); } -TEST_P(QualifiedIdTypeJoinableIndexTest, - InitializationShouldSucceedAfterDestruction) { +TEST_P(QualifiedIdJoinIndexTest, InitializationShouldSucceedAfterDestruction) { const QualifiedIdJoinIndexTestParam& param = GetParam(); { - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); // Insert some data. ICING_ASSERT_OK( @@ -255,10 +250,10 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, // thus initializing another instance on the same files should succeed, and // we should be able to get the same contents. ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); EXPECT_THAT(index, Pointee(SizeIs(3))); EXPECT_THAT(index->Get(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20)), @@ -272,17 +267,17 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, } } -TEST_P(QualifiedIdTypeJoinableIndexTest, +TEST_P(QualifiedIdJoinIndexTest, InitializeExistingFilesWithDifferentMagicShouldFail) { const QualifiedIdJoinIndexTestParam& param = GetParam(); { - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); ICING_ASSERT_OK( index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), /*ref_qualified_id_str=*/"namespace#uriA")); @@ -297,50 +292,49 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, ScopedFd metadata_sfd(filesystem_.OpenForWrite(metadata_file_path.c_str())); ASSERT_THAT(metadata_sfd.is_valid(), IsTrue()); - auto metadata_buffer = std::make_unique<uint8_t[]>( - QualifiedIdTypeJoinableIndex::kMetadataFileSize); - ASSERT_THAT( - filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(), - QualifiedIdTypeJoinableIndex::kMetadataFileSize, - /*offset=*/0), - IsTrue()); + auto metadata_buffer = + std::make_unique<uint8_t[]>(QualifiedIdJoinIndex::kMetadataFileSize); + ASSERT_THAT(filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(), + QualifiedIdJoinIndex::kMetadataFileSize, + /*offset=*/0), + IsTrue()); // Manually change magic and update checksums. Crcs* crcs = reinterpret_cast<Crcs*>( metadata_buffer.get() + - QualifiedIdTypeJoinableIndex::kCrcsMetadataBufferOffset); + QualifiedIdJoinIndex::kCrcsMetadataBufferOffset); Info* info = reinterpret_cast<Info*>( metadata_buffer.get() + - QualifiedIdTypeJoinableIndex::kInfoMetadataBufferOffset); + QualifiedIdJoinIndex::kInfoMetadataBufferOffset); info->magic += kCorruptedValueOffset; crcs->component_crcs.info_crc = info->ComputeChecksum().Get(); crcs->all_crc = crcs->component_crcs.ComputeChecksum().Get(); - ASSERT_THAT(filesystem_.PWrite( - metadata_sfd.get(), /*offset=*/0, metadata_buffer.get(), - QualifiedIdTypeJoinableIndex::kMetadataFileSize), + ASSERT_THAT(filesystem_.PWrite(metadata_sfd.get(), /*offset=*/0, + metadata_buffer.get(), + QualifiedIdJoinIndex::kMetadataFileSize), IsTrue()); } - // Attempt to create the qualified id type joinable index with different - // magic. This should fail. - EXPECT_THAT(QualifiedIdTypeJoinableIndex::Create( - filesystem_, working_path_, param.pre_mapping_fbv, - param.use_persistent_hash_map), + // Attempt to create the qualified id join index with different magic. This + // should fail. + EXPECT_THAT(QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION, HasSubstr("Incorrect magic value"))); } -TEST_P(QualifiedIdTypeJoinableIndexTest, +TEST_P(QualifiedIdJoinIndexTest, InitializeExistingFilesWithWrongAllCrcShouldFail) { const QualifiedIdJoinIndexTestParam& param = GetParam(); { - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); ICING_ASSERT_OK( index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), /*ref_qualified_id_str=*/"namespace#uriA")); @@ -354,46 +348,45 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, ScopedFd metadata_sfd(filesystem_.OpenForWrite(metadata_file_path.c_str())); ASSERT_THAT(metadata_sfd.is_valid(), IsTrue()); - auto metadata_buffer = std::make_unique<uint8_t[]>( - QualifiedIdTypeJoinableIndex::kMetadataFileSize); - ASSERT_THAT( - filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(), - QualifiedIdTypeJoinableIndex::kMetadataFileSize, - /*offset=*/0), - IsTrue()); + auto metadata_buffer = + std::make_unique<uint8_t[]>(QualifiedIdJoinIndex::kMetadataFileSize); + ASSERT_THAT(filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(), + QualifiedIdJoinIndex::kMetadataFileSize, + /*offset=*/0), + IsTrue()); // Manually corrupt all_crc Crcs* crcs = reinterpret_cast<Crcs*>( metadata_buffer.get() + - QualifiedIdTypeJoinableIndex::kCrcsMetadataBufferOffset); + QualifiedIdJoinIndex::kCrcsMetadataBufferOffset); crcs->all_crc += kCorruptedValueOffset; - ASSERT_THAT(filesystem_.PWrite( - metadata_sfd.get(), /*offset=*/0, metadata_buffer.get(), - QualifiedIdTypeJoinableIndex::kMetadataFileSize), + ASSERT_THAT(filesystem_.PWrite(metadata_sfd.get(), /*offset=*/0, + metadata_buffer.get(), + QualifiedIdJoinIndex::kMetadataFileSize), IsTrue()); } - // Attempt to create the qualified id type joinable index with metadata - // containing corrupted all_crc. This should fail. - EXPECT_THAT(QualifiedIdTypeJoinableIndex::Create( - filesystem_, working_path_, param.pre_mapping_fbv, - param.use_persistent_hash_map), + // Attempt to create the qualified id join index with metadata containing + // corrupted all_crc. This should fail. + EXPECT_THAT(QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION, HasSubstr("Invalid all crc"))); } -TEST_P(QualifiedIdTypeJoinableIndexTest, +TEST_P(QualifiedIdJoinIndexTest, InitializeExistingFilesWithCorruptedInfoShouldFail) { const QualifiedIdJoinIndexTestParam& param = GetParam(); { - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); ICING_ASSERT_OK( index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), /*ref_qualified_id_str=*/"namespace#uriA")); @@ -407,47 +400,46 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, ScopedFd metadata_sfd(filesystem_.OpenForWrite(metadata_file_path.c_str())); ASSERT_THAT(metadata_sfd.is_valid(), IsTrue()); - auto metadata_buffer = std::make_unique<uint8_t[]>( - QualifiedIdTypeJoinableIndex::kMetadataFileSize); - ASSERT_THAT( - filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(), - QualifiedIdTypeJoinableIndex::kMetadataFileSize, - /*offset=*/0), - IsTrue()); + auto metadata_buffer = + std::make_unique<uint8_t[]>(QualifiedIdJoinIndex::kMetadataFileSize); + ASSERT_THAT(filesystem_.PRead(metadata_sfd.get(), metadata_buffer.get(), + QualifiedIdJoinIndex::kMetadataFileSize, + /*offset=*/0), + IsTrue()); // Modify info, but don't update the checksum. This would be similar to // corruption of info. Info* info = reinterpret_cast<Info*>( metadata_buffer.get() + - QualifiedIdTypeJoinableIndex::kInfoMetadataBufferOffset); + QualifiedIdJoinIndex::kInfoMetadataBufferOffset); info->last_added_document_id += kCorruptedValueOffset; - ASSERT_THAT(filesystem_.PWrite( - metadata_sfd.get(), /*offset=*/0, metadata_buffer.get(), - QualifiedIdTypeJoinableIndex::kMetadataFileSize), + ASSERT_THAT(filesystem_.PWrite(metadata_sfd.get(), /*offset=*/0, + metadata_buffer.get(), + QualifiedIdJoinIndex::kMetadataFileSize), IsTrue()); } - // Attempt to create the qualified id type joinable index with info that - // doesn't match its checksum. This should fail. - EXPECT_THAT(QualifiedIdTypeJoinableIndex::Create( - filesystem_, working_path_, param.pre_mapping_fbv, - param.use_persistent_hash_map), + // Attempt to create the qualified id join index with info that doesn't match + // its checksum. This should fail. + EXPECT_THAT(QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION, HasSubstr("Invalid info crc"))); } -TEST_P(QualifiedIdTypeJoinableIndexTest, +TEST_P(QualifiedIdJoinIndexTest, InitializeExistingFilesWithCorruptedDocJoinInfoMapperShouldFail) { const QualifiedIdJoinIndexTestParam& param = GetParam(); { - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); ICING_ASSERT_OK( index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), /*ref_qualified_id_str=*/"namespace#uriA")); @@ -478,26 +470,26 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, ASSERT_THAT(old_crc, Not(Eq(new_crc))); } - // Attempt to create the qualified id type joinable index with corrupted + // Attempt to create the qualified id join index with corrupted // doc_join_info_mapper. This should fail. - EXPECT_THAT(QualifiedIdTypeJoinableIndex::Create( - filesystem_, working_path_, param.pre_mapping_fbv, - param.use_persistent_hash_map), + EXPECT_THAT(QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION, HasSubstr("Invalid storages crc"))); } -TEST_P(QualifiedIdTypeJoinableIndexTest, +TEST_P(QualifiedIdJoinIndexTest, InitializeExistingFilesWithCorruptedQualifiedIdStorageShouldFail) { const QualifiedIdJoinIndexTestParam& param = GetParam(); { - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); ICING_ASSERT_OK( index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), /*ref_qualified_id_str=*/"namespace#uriA")); @@ -524,24 +516,24 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, ASSERT_THAT(old_crc, Not(Eq(new_crc))); } - // Attempt to create the qualified id type joinable index with corrupted + // Attempt to create the qualified id join index with corrupted // qualified_id_storage. This should fail. - EXPECT_THAT(QualifiedIdTypeJoinableIndex::Create( - filesystem_, working_path_, param.pre_mapping_fbv, - param.use_persistent_hash_map), + EXPECT_THAT(QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION, HasSubstr("Invalid storages crc"))); } -TEST_P(QualifiedIdTypeJoinableIndexTest, InvalidPut) { +TEST_P(QualifiedIdJoinIndexTest, InvalidPut) { const QualifiedIdJoinIndexTestParam& param = GetParam(); - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); DocJoinInfo default_invalid; EXPECT_THAT( @@ -549,22 +541,22 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, InvalidPut) { StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } -TEST_P(QualifiedIdTypeJoinableIndexTest, InvalidGet) { +TEST_P(QualifiedIdJoinIndexTest, InvalidGet) { const QualifiedIdJoinIndexTestParam& param = GetParam(); - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); DocJoinInfo default_invalid; EXPECT_THAT(index->Get(default_invalid), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } -TEST_P(QualifiedIdTypeJoinableIndexTest, PutAndGet) { +TEST_P(QualifiedIdJoinIndexTest, PutAndGet) { const QualifiedIdJoinIndexTestParam& param = GetParam(); DocJoinInfo target_info1(/*document_id=*/1, /*joinable_property_id=*/20); @@ -577,12 +569,12 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, PutAndGet) { std::string_view ref_qualified_id_str_c = "namespace#uriC"; { - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); EXPECT_THAT(index->Put(target_info1, ref_qualified_id_str_a), IsOk()); EXPECT_THAT(index->Put(target_info2, ref_qualified_id_str_b), IsOk()); @@ -598,29 +590,28 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, PutAndGet) { // Verify we can get all of them after destructing and re-initializing. ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); EXPECT_THAT(index, Pointee(SizeIs(3))); EXPECT_THAT(index->Get(target_info1), IsOkAndHolds(ref_qualified_id_str_a)); EXPECT_THAT(index->Get(target_info2), IsOkAndHolds(ref_qualified_id_str_b)); EXPECT_THAT(index->Get(target_info3), IsOkAndHolds(ref_qualified_id_str_c)); } -TEST_P(QualifiedIdTypeJoinableIndexTest, - GetShouldReturnNotFoundErrorIfNotExist) { +TEST_P(QualifiedIdJoinIndexTest, GetShouldReturnNotFoundErrorIfNotExist) { const QualifiedIdJoinIndexTestParam& param = GetParam(); DocJoinInfo target_info(/*document_id=*/1, /*joinable_property_id=*/20); std::string_view ref_qualified_id_str = "namespace#uriA"; - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); // Verify entry is not found in the beginning. EXPECT_THAT(index->Get(target_info), @@ -636,14 +627,14 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } -TEST_P(QualifiedIdTypeJoinableIndexTest, SetLastAddedDocumentId) { +TEST_P(QualifiedIdJoinIndexTest, SetLastAddedDocumentId) { const QualifiedIdJoinIndexTestParam& param = GetParam(); ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); EXPECT_THAT(index->last_added_document_id(), Eq(kInvalidDocumentId)); @@ -657,15 +648,15 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, SetLastAddedDocumentId) { } TEST_P( - QualifiedIdTypeJoinableIndexTest, + QualifiedIdJoinIndexTest, SetLastAddedDocumentIdShouldIgnoreNewDocumentIdNotGreaterThanTheCurrent) { const QualifiedIdJoinIndexTestParam& param = GetParam(); ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); constexpr DocumentId kDocumentId = 123; index->set_last_added_document_id(kDocumentId); @@ -678,14 +669,14 @@ TEST_P( EXPECT_THAT(index->last_added_document_id(), Eq(kDocumentId)); } -TEST_P(QualifiedIdTypeJoinableIndexTest, Optimize) { +TEST_P(QualifiedIdJoinIndexTest, Optimize) { const QualifiedIdJoinIndexTestParam& param = GetParam(); ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); ICING_ASSERT_OK( index->Put(DocJoinInfo(/*document_id=*/3, /*joinable_property_id=*/10), @@ -759,14 +750,14 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, Optimize) { IsOkAndHolds("namespace#uriD")); } -TEST_P(QualifiedIdTypeJoinableIndexTest, OptimizeOutOfRangeDocumentId) { +TEST_P(QualifiedIdJoinIndexTest, OptimizeOutOfRangeDocumentId) { const QualifiedIdJoinIndexTestParam& param = GetParam(); ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); ICING_ASSERT_OK( index->Put(DocJoinInfo(/*document_id=*/99, /*joinable_property_id=*/10), @@ -788,14 +779,14 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, OptimizeOutOfRangeDocumentId) { EXPECT_THAT(index, Pointee(IsEmpty())); } -TEST_P(QualifiedIdTypeJoinableIndexTest, OptimizeDeleteAll) { +TEST_P(QualifiedIdJoinIndexTest, OptimizeDeleteAll) { const QualifiedIdJoinIndexTestParam& param = GetParam(); ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); ICING_ASSERT_OK( index->Put(DocJoinInfo(/*document_id=*/3, /*joinable_property_id=*/10), @@ -827,19 +818,19 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, OptimizeDeleteAll) { EXPECT_THAT(index, Pointee(IsEmpty())); } -TEST_P(QualifiedIdTypeJoinableIndexTest, Clear) { +TEST_P(QualifiedIdJoinIndexTest, Clear) { const QualifiedIdJoinIndexTestParam& param = GetParam(); DocJoinInfo target_info1(/*document_id=*/1, /*joinable_property_id=*/20); DocJoinInfo target_info2(/*document_id=*/3, /*joinable_property_id=*/5); DocJoinInfo target_info3(/*document_id=*/6, /*joinable_property_id=*/13); - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); ICING_ASSERT_OK( index->Put(target_info1, /*ref_qualified_id_str=*/"namespace#uriA")); ICING_ASSERT_OK( @@ -862,7 +853,7 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, Clear) { EXPECT_THAT(index->Get(target_info3), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); - // Joinable index should be able to work normally after Clear(). + // Join index should be able to work normally after Clear(). DocJoinInfo target_info4(/*document_id=*/2, /*joinable_property_id=*/19); ICING_ASSERT_OK( index->Put(target_info4, /*ref_qualified_id_str=*/"namespace#uriD")); @@ -876,9 +867,9 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, Clear) { // Verify index after reconstructing. ICING_ASSERT_OK_AND_ASSIGN( - index, QualifiedIdTypeJoinableIndex::Create( - filesystem_, working_path_, param.pre_mapping_fbv, - param.use_persistent_hash_map)); + index, QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); EXPECT_THAT(index->last_added_document_id(), Eq(2)); EXPECT_THAT(index->Get(target_info1), StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); @@ -889,16 +880,16 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, Clear) { EXPECT_THAT(index->Get(target_info4), IsOkAndHolds("namespace#uriD")); } -TEST_P(QualifiedIdTypeJoinableIndexTest, SwitchKeyMapperTypeShouldReturnError) { +TEST_P(QualifiedIdJoinIndexTest, SwitchKeyMapperTypeShouldReturnError) { const QualifiedIdJoinIndexTestParam& param = GetParam(); { - // Create new qualified id type joinable index + // Create new qualified id join index ICING_ASSERT_OK_AND_ASSIGN( - std::unique_ptr<QualifiedIdTypeJoinableIndex> index, - QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - param.use_persistent_hash_map)); + std::unique_ptr<QualifiedIdJoinIndex> index, + QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + param.use_persistent_hash_map)); ICING_ASSERT_OK( index->Put(DocJoinInfo(/*document_id=*/1, /*joinable_property_id=*/20), /*ref_qualified_id_str=*/"namespace#uriA")); @@ -907,14 +898,14 @@ TEST_P(QualifiedIdTypeJoinableIndexTest, SwitchKeyMapperTypeShouldReturnError) { } bool switch_key_mapper_flag = !param.use_persistent_hash_map; - EXPECT_THAT(QualifiedIdTypeJoinableIndex::Create(filesystem_, working_path_, - param.pre_mapping_fbv, - switch_key_mapper_flag), + EXPECT_THAT(QualifiedIdJoinIndex::Create(filesystem_, working_path_, + param.pre_mapping_fbv, + switch_key_mapper_flag), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } INSTANTIATE_TEST_SUITE_P( - QualifiedIdTypeJoinableIndexTest, QualifiedIdTypeJoinableIndexTest, + QualifiedIdJoinIndexTest, QualifiedIdJoinIndexTest, testing::Values( QualifiedIdJoinIndexTestParam(/*pre_mapping_fbv_in=*/true, /*use_persistent_hash_map_in=*/true), diff --git a/icing/join/qualified-id-join-indexing-handler.cc b/icing/join/qualified-id-join-indexing-handler.cc index 86af043..344cf41 100644 --- a/icing/join/qualified-id-join-indexing-handler.cc +++ b/icing/join/qualified-id-join-indexing-handler.cc @@ -21,7 +21,7 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/join/doc-join-info.h" -#include "icing/join/qualified-id-type-joinable-index.h" +#include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id.h" #include "icing/legacy/core/icing-string-util.h" #include "icing/proto/logging.pb.h" @@ -38,7 +38,7 @@ namespace lib { /* static */ libtextclassifier3::StatusOr< std::unique_ptr<QualifiedIdJoinIndexingHandler>> QualifiedIdJoinIndexingHandler::Create( - const Clock* clock, QualifiedIdTypeJoinableIndex* qualified_id_join_index) { + const Clock* clock, QualifiedIdJoinIndex* qualified_id_join_index) { ICING_RETURN_ERROR_IF_NULL(clock); ICING_RETURN_ERROR_IF_NULL(qualified_id_join_index); diff --git a/icing/join/qualified-id-join-indexing-handler.h b/icing/join/qualified-id-join-indexing-handler.h index 434403e..f44e45d 100644 --- a/icing/join/qualified-id-join-indexing-handler.h +++ b/icing/join/qualified-id-join-indexing-handler.h @@ -17,7 +17,7 @@ #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/index/data-indexing-handler.h" -#include "icing/join/qualified-id-type-joinable-index.h" +#include "icing/join/qualified-id-join-index.h" #include "icing/proto/logging.pb.h" #include "icing/store/document-id.h" #include "icing/util/clock.h" @@ -37,13 +37,12 @@ class QualifiedIdJoinIndexingHandler : public DataIndexingHandler { // - FAILED_PRECONDITION_ERROR if any of the input pointer is null static libtextclassifier3::StatusOr< std::unique_ptr<QualifiedIdJoinIndexingHandler>> - Create(const Clock* clock, - QualifiedIdTypeJoinableIndex* qualified_id_join_index); + Create(const Clock* clock, QualifiedIdJoinIndex* qualified_id_join_index); ~QualifiedIdJoinIndexingHandler() override = default; // Handles the joinable qualified id data indexing process: add data into the - // qualified id type joinable cache. + // qualified id join index. // /// Returns: // - OK on success. @@ -51,18 +50,18 @@ class QualifiedIdJoinIndexingHandler : public DataIndexingHandler { // than or equal to the document_id of a previously indexed document in // non recovery mode. // - INTERNAL_ERROR if any other errors occur. - // - Any QualifiedIdTypeJoinableIndex errors. + // - Any QualifiedIdJoinIndex errors. libtextclassifier3::Status Handle( const TokenizedDocument& tokenized_document, DocumentId document_id, bool recovery_mode, PutDocumentStatsProto* put_document_stats) override; private: explicit QualifiedIdJoinIndexingHandler( - const Clock* clock, QualifiedIdTypeJoinableIndex* qualified_id_join_index) + const Clock* clock, QualifiedIdJoinIndex* qualified_id_join_index) : DataIndexingHandler(clock), qualified_id_join_index_(*qualified_id_join_index) {} - QualifiedIdTypeJoinableIndex& qualified_id_join_index_; // Does not own. + QualifiedIdJoinIndex& qualified_id_join_index_; // Does not own. }; } // namespace lib diff --git a/icing/join/qualified-id-join-indexing-handler_test.cc b/icing/join/qualified-id-join-indexing-handler_test.cc index e48dc33..7e89dfa 100644 --- a/icing/join/qualified-id-join-indexing-handler_test.cc +++ b/icing/join/qualified-id-join-indexing-handler_test.cc @@ -23,7 +23,7 @@ #include "gtest/gtest.h" #include "icing/document-builder.h" #include "icing/file/filesystem.h" -#include "icing/join/qualified-id-type-joinable-index.h" +#include "icing/join/qualified-id-join-index.h" #include "icing/join/qualified-id.h" #include "icing/portable/platform.h" #include "icing/proto/document.pb.h" @@ -92,9 +92,9 @@ class QualifiedIdJoinIndexingHandlerTest : public ::testing::Test { ICING_ASSERT_OK_AND_ASSIGN( qualified_id_join_index_, - QualifiedIdTypeJoinableIndex::Create( - filesystem_, qualified_id_join_index_dir_, - /*pre_mapping_fbv=*/false, /*use_persistent_hash_map=*/false)); + QualifiedIdJoinIndex::Create(filesystem_, qualified_id_join_index_dir_, + /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false)); language_segmenter_factory::SegmenterOptions segmenter_options(ULOC_US); ICING_ASSERT_OK_AND_ASSIGN( @@ -156,7 +156,7 @@ class QualifiedIdJoinIndexingHandlerTest : public ::testing::Test { std::string qualified_id_join_index_dir_; std::string schema_store_dir_; - std::unique_ptr<QualifiedIdTypeJoinableIndex> qualified_id_join_index_; + std::unique_ptr<QualifiedIdJoinIndex> qualified_id_join_index_; std::unique_ptr<LanguageSegmenter> lang_segmenter_; std::unique_ptr<SchemaStore> schema_store_; }; diff --git a/icing/performance-configuration.cc b/icing/performance-configuration.cc index 07ff9bc..1518381 100644 --- a/icing/performance-configuration.cc +++ b/icing/performance-configuration.cc @@ -38,20 +38,17 @@ namespace { // rendering 2 frames. // // With the information above, we then try to choose default values for -// query_length and num_to_score so that the overall time can comfortably fit -// in with our goal. +// query_length so that the overall time can comfortably fit in with our goal +// (note that num_to_score will be decided by the client, which is specified in +// ResultSpecProto). // 1. Set query_length to 23000 so that any query can be executed by // QueryProcessor within 15 ms on a Pixel 3 XL according to results of // //icing/query:query-processor_benchmark. -// 2. Set num_to_score to 30000 so that results can be scored and ranked within -// 3 ms on a Pixel 3 XL according to results of -// //icing/scoring:score-and-rank_benchmark. // // In the worse-case scenario, we still have [33 ms - 15 ms - 3 ms] = 15 ms left // for all the other things like proto parsing, document fetching, and even // Android Binder calls if Icing search engine runs in a separate process. constexpr int kMaxQueryLength = 23000; -constexpr int kDefaultNumToScore = 30000; // New Android devices nowadays all allow more than 16 MB memory per app. Using // that as a guideline and being more conservative, we set 4 MB as the safe @@ -67,8 +64,7 @@ constexpr int kMaxNumTotalHits = kSafeMemoryUsage / sizeof(ScoredDocumentHit); } // namespace PerformanceConfiguration::PerformanceConfiguration() - : PerformanceConfiguration(kMaxQueryLength, kDefaultNumToScore, - kMaxNumTotalHits) {} + : PerformanceConfiguration(kMaxQueryLength, kMaxNumTotalHits) {} } // namespace lib } // namespace icing diff --git a/icing/performance-configuration.h b/icing/performance-configuration.h index b9282ca..3ec67f3 100644 --- a/icing/performance-configuration.h +++ b/icing/performance-configuration.h @@ -23,10 +23,8 @@ struct PerformanceConfiguration { // Loads default configuration. PerformanceConfiguration(); - PerformanceConfiguration(int max_query_length_in, int num_to_score_in, - int max_num_total_hits) + PerformanceConfiguration(int max_query_length_in, int max_num_total_hits) : max_query_length(max_query_length_in), - num_to_score(num_to_score_in), max_num_total_hits(max_num_total_hits) {} // Search performance @@ -34,9 +32,6 @@ struct PerformanceConfiguration { // Maximum length of query to execute in IndexProcessor. int max_query_length; - // Number of results to score in ScoringProcessor for every query. - int num_to_score; - // Memory // Maximum number of ScoredDocumentHits to cache in the ResultStateManager at diff --git a/icing/query/advanced_query_parser/query-visitor_test.cc b/icing/query/advanced_query_parser/query-visitor_test.cc index 0d7ba6d..59e924d 100644 --- a/icing/query/advanced_query_parser/query-visitor_test.cc +++ b/icing/query/advanced_query_parser/query-visitor_test.cc @@ -114,17 +114,20 @@ class QueryVisitorTest : public ::testing::TestWithParam<QueryType> { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, store_dir_, &clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, store_dir_, &clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); Index::Options options(index_dir_.c_str(), - /*index_merge_size=*/1024 * 1024); + /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, &icing_filesystem_)); diff --git a/icing/query/query-processor.cc b/icing/query/query-processor.cc index 5e0b696..3e43ad9 100644 --- a/icing/query/query-processor.cc +++ b/icing/query/query-processor.cc @@ -176,6 +176,13 @@ libtextclassifier3::StatusOr<QueryResults> QueryProcessor::ParseSearch( results.root_iterator = std::make_unique<DocHitInfoIteratorFilter>( std::move(results.root_iterator), &document_store_, &schema_store_, options, current_time_ms); + // TODO(b/294114230): Move this SectionRestrict filter from root level to + // lower levels if that would improve performance. + if (!search_spec.type_property_filters().empty()) { + results.root_iterator = std::make_unique<DocHitInfoIteratorSectionRestrict>( + std::move(results.root_iterator), &document_store_, &schema_store_, + search_spec, current_time_ms); + } return results; } diff --git a/icing/query/query-processor_benchmark.cc b/icing/query/query-processor_benchmark.cc index 89f3b54..025e8e6 100644 --- a/icing/query/query-processor_benchmark.cc +++ b/icing/query/query-processor_benchmark.cc @@ -81,7 +81,9 @@ void AddTokenToIndex(Index* index, DocumentId document_id, SectionId section_id, std::unique_ptr<Index> CreateIndex(const IcingFilesystem& icing_filesystem, const Filesystem& filesystem, const std::string& index_dir) { - Index::Options options(index_dir, /*index_merge_size=*/1024 * 1024 * 10); + Index::Options options(index_dir, /*index_merge_size=*/1024 * 1024 * 10, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); return Index::Create(options, &filesystem, &icing_filesystem).ValueOrDie(); } @@ -98,7 +100,8 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( return DocumentStore::Create( filesystem, base_dir, clock, schema_store, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, /*initialize_stats=*/nullptr); } diff --git a/icing/query/query-processor_test.cc b/icing/query/query-processor_test.cc index 3d3cf48..e64de32 100644 --- a/icing/query/query-processor_test.cc +++ b/icing/query/query-processor_test.cc @@ -69,7 +69,8 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( return DocumentStore::Create( filesystem, base_dir, clock, schema_store, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, /*initialize_stats=*/nullptr); } @@ -112,7 +113,9 @@ class QueryProcessorTest document_store_ = std::move(create_result.document_store); Index::Options options(index_dir_, - /*index_merge_size=*/1024 * 1024); + /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, &icing_filesystem_)); // TODO(b/249829533): switch to use persistent numeric index. @@ -2645,6 +2648,261 @@ TEST_P(QueryProcessorTest, PropertyFilterTermAndUnrestrictedTerm) { EXPECT_THAT(results.query_terms["foo"], UnorderedElementsAre("animal")); } +TEST_P(QueryProcessorTest, TypePropertyFilter) { + // Create the schema and document store + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email") + .AddProperty( + PropertyConfigBuilder() + .SetName("foo") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("bar") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("baz") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder().SetType("message") + .AddProperty( + PropertyConfigBuilder() + .SetName("foo") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("bar") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("baz") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + // SectionIds are assigned in ascending order per schema type, + // alphabetically. + int email_bar_section_id = 0; + int email_baz_section_id = 1; + int email_foo_section_id = 2; + int message_bar_section_id = 0; + int message_baz_section_id = 1; + int message_foo_section_id = 2; + ASSERT_THAT(schema_store_->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOk()); + + // These documents don't actually match to the tokens in the index. We're + // inserting the documents to get the appropriate number of documents and + // schema types populated. + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email_document_id, + document_store_->Put(DocumentBuilder() + .SetKey("namespace", "1") + .SetSchema("email") + .Build())); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message_document_id, + document_store_->Put(DocumentBuilder() + .SetKey("namespace", "2") + .SetSchema("message") + .Build())); + + // Poplate the index + TermMatchType::Code term_match_type = TermMatchType::EXACT_ONLY; + + // Email document has content "animal" in all sections + ASSERT_THAT(AddTokenToIndex(email_document_id, email_foo_section_id, + term_match_type, "animal"), + IsOk()); + ASSERT_THAT(AddTokenToIndex(email_document_id, email_bar_section_id, + term_match_type, "animal"), + IsOk()); + ASSERT_THAT(AddTokenToIndex(email_document_id, email_baz_section_id, + term_match_type, "animal"), + IsOk()); + + // Message document has content "animal" in all sections + ASSERT_THAT(AddTokenToIndex(message_document_id, message_foo_section_id, + term_match_type, "animal"), + IsOk()); + ASSERT_THAT(AddTokenToIndex(message_document_id, message_bar_section_id, + term_match_type, "animal"), + IsOk()); + ASSERT_THAT(AddTokenToIndex(message_document_id, message_baz_section_id, + term_match_type, "animal"), + IsOk()); + + SearchSpecProto search_spec; + search_spec.set_query("animal"); + search_spec.set_term_match_type(term_match_type); + search_spec.set_search_type(GetParam()); + + // email has property filters for foo and baz properties + TypePropertyMask *email_mask = search_spec.add_type_property_filters(); + email_mask->set_schema_type("email"); + email_mask->add_paths("foo"); + email_mask->add_paths("baz"); + + // message has property filters for bar and baz properties + TypePropertyMask *message_mask = search_spec.add_type_property_filters(); + message_mask->set_schema_type("message"); + message_mask->add_paths("bar"); + message_mask->add_paths("baz"); + + ICING_ASSERT_OK_AND_ASSIGN( + QueryResults results, + query_processor_->ParseSearch( + search_spec, ScoringSpecProto::RankingStrategy::RELEVANCE_SCORE, + fake_clock_.GetSystemTimeMilliseconds())); + + // Ordered by descending DocumentId, so message comes first since it was + // inserted last + DocHitInfo expected_doc_hit_info1(message_document_id); + expected_doc_hit_info1.UpdateSection(message_bar_section_id); + expected_doc_hit_info1.UpdateSection(message_baz_section_id); + DocHitInfo expected_doc_hit_info2(email_document_id); + expected_doc_hit_info2.UpdateSection(email_foo_section_id); + expected_doc_hit_info2.UpdateSection(email_baz_section_id); + EXPECT_THAT(GetDocHitInfos(results.root_iterator.get()), + ElementsAre(expected_doc_hit_info1, expected_doc_hit_info2)); + EXPECT_THAT(results.query_term_iterators, SizeIs(1)); + + EXPECT_THAT(results.query_terms, SizeIs(1)); + EXPECT_THAT(results.query_terms[""], UnorderedElementsAre("animal")); +} + +TEST_P(QueryProcessorTest, TypePropertyFilterWithSectionRestrict) { + // Create the schema and document store + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder().SetType("email") + .AddProperty( + PropertyConfigBuilder() + .SetName("foo") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("bar") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("baz") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder().SetType("message") + .AddProperty( + PropertyConfigBuilder() + .SetName("foo") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("bar") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("baz") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + // SectionIds are assigned in ascending order per schema type, + // alphabetically. + int email_bar_section_id = 0; + int email_baz_section_id = 1; + int email_foo_section_id = 2; + int message_bar_section_id = 0; + int message_baz_section_id = 1; + int message_foo_section_id = 2; + ASSERT_THAT(schema_store_->SetSchema( + schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOk()); + + // These documents don't actually match to the tokens in the index. We're + // inserting the documents to get the appropriate number of documents and + // schema types populated. + ICING_ASSERT_OK_AND_ASSIGN(DocumentId email_document_id, + document_store_->Put(DocumentBuilder() + .SetKey("namespace", "1") + .SetSchema("email") + .Build())); + ICING_ASSERT_OK_AND_ASSIGN(DocumentId message_document_id, + document_store_->Put(DocumentBuilder() + .SetKey("namespace", "2") + .SetSchema("message") + .Build())); + + // Poplate the index + TermMatchType::Code term_match_type = TermMatchType::EXACT_ONLY; + + // Email document has content "animal" in all sections + ASSERT_THAT(AddTokenToIndex(email_document_id, email_foo_section_id, + term_match_type, "animal"), + IsOk()); + ASSERT_THAT(AddTokenToIndex(email_document_id, email_bar_section_id, + term_match_type, "animal"), + IsOk()); + ASSERT_THAT(AddTokenToIndex(email_document_id, email_baz_section_id, + term_match_type, "animal"), + IsOk()); + + // Message document has content "animal" in all sections + ASSERT_THAT(AddTokenToIndex(message_document_id, message_foo_section_id, + term_match_type, "animal"), + IsOk()); + ASSERT_THAT(AddTokenToIndex(message_document_id, message_bar_section_id, + term_match_type, "animal"), + IsOk()); + ASSERT_THAT(AddTokenToIndex(message_document_id, message_baz_section_id, + term_match_type, "animal"), + IsOk()); + + SearchSpecProto search_spec; + // Create a section filter '<section name>:<query term>' + search_spec.set_query("foo:animal"); + search_spec.set_term_match_type(term_match_type); + search_spec.set_search_type(GetParam()); + + // email has property filters for foo and baz properties + TypePropertyMask *email_mask = search_spec.add_type_property_filters(); + email_mask->set_schema_type("email"); + email_mask->add_paths("foo"); + email_mask->add_paths("baz"); + + // message has property filters for bar and baz properties + TypePropertyMask *message_mask = search_spec.add_type_property_filters(); + message_mask->set_schema_type("message"); + message_mask->add_paths("bar"); + message_mask->add_paths("baz"); + + ICING_ASSERT_OK_AND_ASSIGN( + QueryResults results, + query_processor_->ParseSearch( + search_spec, ScoringSpecProto::RankingStrategy::RELEVANCE_SCORE, + fake_clock_.GetSystemTimeMilliseconds())); + + // Only hits in sections allowed by both the property filters and section + // restricts should be returned. Message document should not be returned since + // section foo specified in the section restrict is not allowed by the + // property filters. + DocHitInfo expected_doc_hit_info(email_document_id); + expected_doc_hit_info.UpdateSection(email_foo_section_id); + EXPECT_THAT(GetDocHitInfos(results.root_iterator.get()), + ElementsAre(expected_doc_hit_info)); + EXPECT_THAT(results.query_term_iterators, SizeIs(1)); + + EXPECT_THAT(results.query_terms, SizeIs(1)); + EXPECT_THAT(results.query_terms["foo"], UnorderedElementsAre("animal")); +} + TEST_P(QueryProcessorTest, DocumentBeforeTtlNotFilteredOut) { // Create the schema and document store SchemaProto schema = SchemaBuilder() diff --git a/icing/query/suggestion-processor_test.cc b/icing/query/suggestion-processor_test.cc index b1336b3..9f9094d 100644 --- a/icing/query/suggestion-processor_test.cc +++ b/icing/query/suggestion-processor_test.cc @@ -85,17 +85,20 @@ class SuggestionProcessorTest : public Test { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, store_dir_, &fake_clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, store_dir_, &fake_clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); Index::Options options(index_dir_, - /*index_merge_size=*/1024 * 1024); + /*index_merge_size=*/1024 * 1024, + /*lite_index_sort_at_indexing=*/true, + /*lite_index_sort_size=*/1024 * 8); ICING_ASSERT_OK_AND_ASSIGN( index_, Index::Create(options, &filesystem_, &icing_filesystem_)); // TODO(b/249829533): switch to use persistent numeric index. diff --git a/icing/result/result-retriever-v2_group-result-limiter_test.cc b/icing/result/result-retriever-v2_group-result-limiter_test.cc index 5d8b589..2914a8d 100644 --- a/icing/result/result-retriever-v2_group-result-limiter_test.cc +++ b/icing/result/result-retriever-v2_group-result-limiter_test.cc @@ -89,13 +89,14 @@ class ResultRetrieverV2GroupResultLimiterTest : public testing::Test { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, test_dir_, &fake_clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); } diff --git a/icing/result/result-retriever-v2_projection_test.cc b/icing/result/result-retriever-v2_projection_test.cc index 6b868a5..1a75631 100644 --- a/icing/result/result-retriever-v2_projection_test.cc +++ b/icing/result/result-retriever-v2_projection_test.cc @@ -184,13 +184,14 @@ class ResultRetrieverV2ProjectionTest : public testing::Test { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, test_dir_, &fake_clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); } diff --git a/icing/result/result-retriever-v2_snippet_test.cc b/icing/result/result-retriever-v2_snippet_test.cc index 27f16a0..440d31c 100644 --- a/icing/result/result-retriever-v2_snippet_test.cc +++ b/icing/result/result-retriever-v2_snippet_test.cc @@ -109,13 +109,14 @@ class ResultRetrieverV2SnippetTest : public testing::Test { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, test_dir_, &fake_clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, test_dir_, &fake_clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); } diff --git a/icing/result/result-retriever-v2_test.cc b/icing/result/result-retriever-v2_test.cc index 889dc60..0bd40cc 100644 --- a/icing/result/result-retriever-v2_test.cc +++ b/icing/result/result-retriever-v2_test.cc @@ -220,7 +220,8 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( return DocumentStore::Create( filesystem, base_dir, clock, schema_store, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, /*initialize_stats=*/nullptr); } diff --git a/icing/result/result-state-manager_test.cc b/icing/result/result-state-manager_test.cc index 38d67e8..75d1d93 100644 --- a/icing/result/result-state-manager_test.cc +++ b/icing/result/result-state-manager_test.cc @@ -107,13 +107,14 @@ class ResultStateManagerTest : public testing::Test { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult result, - DocumentStore::Create(&filesystem_, test_dir_, clock_.get(), - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, test_dir_, clock_.get(), schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(result.document_store); ICING_ASSERT_OK_AND_ASSIGN( diff --git a/icing/result/result-state-manager_thread-safety_test.cc b/icing/result/result-state-manager_thread-safety_test.cc index 53745e6..7e7e13c 100644 --- a/icing/result/result-state-manager_thread-safety_test.cc +++ b/icing/result/result-state-manager_thread-safety_test.cc @@ -100,13 +100,14 @@ class ResultStateManagerThreadSafetyTest : public testing::Test { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult result, - DocumentStore::Create(&filesystem_, test_dir_, clock_.get(), - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, test_dir_, clock_.get(), schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(result.document_store); ICING_ASSERT_OK_AND_ASSIGN( diff --git a/icing/result/result-state-v2_test.cc b/icing/result/result-state-v2_test.cc index ab29d6e..0f88023 100644 --- a/icing/result/result-state-v2_test.cc +++ b/icing/result/result-state-v2_test.cc @@ -76,13 +76,14 @@ class ResultStateV2Test : public ::testing::Test { filesystem_.CreateDirectoryRecursively(doc_store_base_dir_.c_str()); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult result, - DocumentStore::Create(&filesystem_, doc_store_base_dir_, &clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, doc_store_base_dir_, &clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(result.document_store); num_total_hits_ = 0; diff --git a/icing/schema-builder.h b/icing/schema-builder.h index e8be483..c74505e 100644 --- a/icing/schema-builder.h +++ b/icing/schema-builder.h @@ -127,6 +127,22 @@ class PropertyConfigBuilder { property_.set_schema_type(std::string(schema_type)); property_.mutable_document_indexing_config()->set_index_nested_properties( index_nested_properties); + property_.mutable_document_indexing_config() + ->clear_indexable_nested_properties_list(); + return *this; + } + + PropertyConfigBuilder& SetDataTypeDocument( + std::string_view schema_type, + std::initializer_list<std::string> indexable_nested_properties_list) { + property_.set_data_type(PropertyConfigProto::DataType::DOCUMENT); + property_.set_schema_type(std::string(schema_type)); + property_.mutable_document_indexing_config()->set_index_nested_properties( + false); + for (const std::string& property : indexable_nested_properties_list) { + property_.mutable_document_indexing_config() + ->add_indexable_nested_properties_list(property); + } return *this; } diff --git a/icing/schema/backup-schema-producer_test.cc b/icing/schema/backup-schema-producer_test.cc index b0e793c..dbd033f 100644 --- a/icing/schema/backup-schema-producer_test.cc +++ b/icing/schema/backup-schema-producer_test.cc @@ -36,6 +36,8 @@ namespace lib { namespace { using ::testing::Eq; +using ::testing::Pointee; +using ::testing::SizeIs; class BackupSchemaProducerTest : public ::testing::Test { protected: @@ -442,6 +444,96 @@ TEST_F(BackupSchemaProducerTest, MakeExtraDocumentIndexedPropertiesUnindexed) { EXPECT_THAT(backup, portable_equals_proto::EqualsProto(expected_backup)); } +TEST_F( + BackupSchemaProducerTest, + MakeExtraDocumentIndexedPropertiesWithIndexableNestedPropertiesListUnindexed) { + PropertyConfigBuilder indexed_string_property_builder = + PropertyConfigBuilder() + .SetCardinality(CARDINALITY_OPTIONAL) + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN); + PropertyConfigBuilder indexed_int_property_builder = + PropertyConfigBuilder() + .SetCardinality(CARDINALITY_OPTIONAL) + .SetDataTypeInt64(NUMERIC_MATCH_RANGE); + SchemaTypeConfigProto typeB = + SchemaTypeConfigBuilder() + .SetType("TypeB") + .AddProperty(indexed_string_property_builder.SetName("prop0")) + .AddProperty(indexed_int_property_builder.SetName("prop1")) + .AddProperty(indexed_string_property_builder.SetName("prop2")) + .AddProperty(indexed_int_property_builder.SetName("prop3")) + .AddProperty(indexed_string_property_builder.SetName("prop4")) + .AddProperty(indexed_int_property_builder.SetName("prop5")) + .AddProperty(indexed_string_property_builder.SetName("prop6")) + .AddProperty(indexed_int_property_builder.SetName("prop7")) + .AddProperty(indexed_string_property_builder.SetName("prop8")) + .AddProperty(indexed_int_property_builder.SetName("prop9")) + .Build(); + + // Create indexed document property by using indexable nested properties list. + PropertyConfigBuilder indexed_document_property_with_list_builder = + PropertyConfigBuilder() + .SetCardinality(CARDINALITY_OPTIONAL) + .SetDataTypeDocument( + "TypeB", /*indexable_nested_properties_list=*/{ + "prop0", "prop1", "prop2", "prop3", "prop4", "prop5", + "unknown1", "unknown2", "unknown3"}); + SchemaTypeConfigProto typeA = + SchemaTypeConfigBuilder() + .SetType("TypeA") + .AddProperty( + indexed_document_property_with_list_builder.SetName("propA")) + .AddProperty( + indexed_document_property_with_list_builder.SetName("propB")) + .Build(); + + SchemaProto schema = SchemaBuilder().AddType(typeA).AddType(typeB).Build(); + + SchemaUtil::TypeConfigMap type_config_map; + SchemaUtil::BuildTypeConfigMap(schema, &type_config_map); + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<DynamicTrieKeyMapper<SchemaTypeId>> type_id_mapper, + DynamicTrieKeyMapper<SchemaTypeId>::Create(filesystem_, schema_store_dir_, + /*maximum_size_bytes=*/10000)); + ASSERT_THAT(type_id_mapper->Put("TypeA", 0), IsOk()); + ASSERT_THAT(type_id_mapper->Put("TypeB", 1), IsOk()); + + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaTypeManager> schema_type_manager, + SchemaTypeManager::Create(type_config_map, type_id_mapper.get())); + ASSERT_THAT(schema_type_manager->section_manager().GetMetadataList("TypeA"), + IsOkAndHolds(Pointee(SizeIs(18)))); + + ICING_ASSERT_OK_AND_ASSIGN( + BackupSchemaProducer backup_producer, + BackupSchemaProducer::Create(schema, + schema_type_manager->section_manager())); + EXPECT_THAT(backup_producer.is_backup_necessary(), Eq(true)); + SchemaProto backup = std::move(backup_producer).Produce(); + + PropertyConfigProto unindexed_document_property = + PropertyConfigBuilder() + .SetCardinality(CARDINALITY_OPTIONAL) + .SetDataType(TYPE_DOCUMENT) + .Build(); + unindexed_document_property.set_schema_type("TypeB"); + PropertyConfigBuilder unindexed_document_property_builder( + unindexed_document_property); + + // "propA" and "propB" both have 9 sections respectively, so we have to drop + // "propB" indexing config to make total # of sections <= 16. + SchemaTypeConfigProto expected_typeA = + SchemaTypeConfigBuilder() + .SetType("TypeA") + .AddProperty( + indexed_document_property_with_list_builder.SetName("propA")) + .AddProperty(unindexed_document_property_builder.SetName("propB")) + .Build(); + SchemaProto expected_backup = + SchemaBuilder().AddType(expected_typeA).AddType(typeB).Build(); + EXPECT_THAT(backup, portable_equals_proto::EqualsProto(expected_backup)); +} + TEST_F(BackupSchemaProducerTest, MakeRfcPropertiesUnindexedFirst) { PropertyConfigBuilder indexed_string_property_builder = PropertyConfigBuilder() @@ -539,31 +631,33 @@ TEST_F(BackupSchemaProducerTest, MakeExtraPropertiesUnindexedMultipleTypes) { .AddProperty(indexed_string_property_builder.SetName("prop2")) .AddProperty(indexed_int_property_builder.SetName("prop3")) .AddProperty(indexed_string_property_builder.SetName("prop4")) - .AddProperty(indexed_int_property_builder.SetName("prop5")) - .AddProperty(indexed_string_property_builder.SetName("prop6")) - .AddProperty(indexed_int_property_builder.SetName("prop7")) - .AddProperty(indexed_string_property_builder.SetName("prop8")) - .AddProperty(indexed_int_property_builder.SetName("prop9")) .Build(); PropertyConfigBuilder indexed_document_property_builder = PropertyConfigBuilder() .SetCardinality(CARDINALITY_OPTIONAL) .SetDataTypeDocument("TypeB", /*index_nested_properties=*/true); + PropertyConfigBuilder indexed_document_property_with_list_builder = + PropertyConfigBuilder() + .SetCardinality(CARDINALITY_OPTIONAL) + .SetDataTypeDocument( + "TypeB", /*indexable_nested_properties_list=*/{ + "prop0", "prop4", "unknown1", "unknown2", "unknown3"}); SchemaTypeConfigProto typeA = SchemaTypeConfigBuilder() .SetType("TypeA") .AddProperty(indexed_string_property_builder.SetName("propA")) - .AddProperty(indexed_int_property_builder.SetName("propB")) + .AddProperty( + indexed_document_property_with_list_builder.SetName("propB")) .AddProperty(indexed_string_property_builder.SetName("propC")) - .AddProperty(indexed_int_property_builder.SetName("propD")) + .AddProperty(indexed_document_property_builder.SetName("propD")) .AddProperty(indexed_string_property_builder.SetName("propE")) .AddProperty(indexed_int_property_builder.SetName("propF")) - .AddProperty(indexed_string_property_builder.SetName("propG")) - .AddProperty(indexed_int_property_builder.SetName("propH")) - .AddProperty(indexed_document_property_builder.SetName("propI")) - .AddProperty(indexed_string_property_builder.SetName("propJ")) - .AddProperty(indexed_int_property_builder.SetName("propK")) + .AddProperty(indexed_document_property_builder.SetName("propG")) + .AddProperty(indexed_string_property_builder.SetName("propH")) + .AddProperty(indexed_int_property_builder.SetName("propI")) + .AddProperty( + indexed_document_property_with_list_builder.SetName("propJ")) .Build(); SchemaProto schema = SchemaBuilder().AddType(typeA).AddType(typeB).Build(); @@ -580,6 +674,8 @@ TEST_F(BackupSchemaProducerTest, MakeExtraPropertiesUnindexedMultipleTypes) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaTypeManager> schema_type_manager, SchemaTypeManager::Create(type_config_map, type_id_mapper.get())); + ASSERT_THAT(schema_type_manager->section_manager().GetMetadataList("TypeA"), + IsOkAndHolds(Pointee(SizeIs(26)))); ICING_ASSERT_OK_AND_ASSIGN( BackupSchemaProducer backup_producer, @@ -605,20 +701,30 @@ TEST_F(BackupSchemaProducerTest, MakeExtraPropertiesUnindexedMultipleTypes) { PropertyConfigBuilder unindexed_document_property_builder( unindexed_document_property); + // On version 0 (Android T): + // - Only "propA", "propC", "propD.prop0", "propD.prop1", "propD.prop2", + // "propD.prop3", "propD.prop4", "propE", "propF" will be assigned sections. + // - Unlike version 2, "propB.prop0", "propB.prop4", "propB.unknown1", + // "propB.unknown2", "propB.unknown3" will be ignored because version 0 + // doesn't recognize indexable nested properties list. + // - So there will be only 9 sections on version 0. We still have potential to + // avoid dropping "propG", "propH", "propI" indexing configs on version 0 + // (in this case it will be 16 sections), but it is ok to make it simple as + // long as total # of sections <= 16. SchemaTypeConfigProto expected_typeA = SchemaTypeConfigBuilder() .SetType("TypeA") .AddProperty(indexed_string_property_builder.SetName("propA")) - .AddProperty(indexed_int_property_builder.SetName("propB")) + .AddProperty( + indexed_document_property_with_list_builder.SetName("propB")) .AddProperty(indexed_string_property_builder.SetName("propC")) - .AddProperty(indexed_int_property_builder.SetName("propD")) + .AddProperty(indexed_document_property_builder.SetName("propD")) .AddProperty(indexed_string_property_builder.SetName("propE")) .AddProperty(indexed_int_property_builder.SetName("propF")) - .AddProperty(indexed_string_property_builder.SetName("propG")) - .AddProperty(indexed_int_property_builder.SetName("propH")) - .AddProperty(unindexed_document_property_builder.SetName("propI")) - .AddProperty(unindexed_string_property_builder.SetName("propJ")) - .AddProperty(unindexed_int_property_builder.SetName("propK")) + .AddProperty(unindexed_document_property_builder.SetName("propG")) + .AddProperty(unindexed_string_property_builder.SetName("propH")) + .AddProperty(unindexed_int_property_builder.SetName("propI")) + .AddProperty(unindexed_document_property_builder.SetName("propJ")) .Build(); SchemaProto expected_backup = SchemaBuilder().AddType(expected_typeA).AddType(typeB).Build(); diff --git a/icing/schema/property-util.cc b/icing/schema/property-util.cc index 7370328..67ff748 100644 --- a/icing/schema/property-util.cc +++ b/icing/schema/property-util.cc @@ -16,11 +16,9 @@ #include <string> #include <string_view> -#include <utility> #include <vector> #include "icing/text_classifier/lib3/utils/base/statusor.h" -#include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" #include "icing/absl_ports/str_join.h" #include "icing/proto/document.pb.h" @@ -85,6 +83,23 @@ std::vector<PropertyInfo> ParsePropertyPathExpr( return property_infos; } +bool IsParentPropertyPath(std::string_view property_path_expr1, + std::string_view property_path_expr2) { + if (property_path_expr2.length() < property_path_expr1.length()) { + return false; + } + if (property_path_expr1 != + property_path_expr2.substr(0, property_path_expr1.length())) { + return false; + } + if (property_path_expr2.length() > property_path_expr1.length() && + property_path_expr2[property_path_expr1.length()] != + kPropertyPathSeparator[0]) { + return false; + } + return true; +} + const PropertyProto* GetPropertyProto(const DocumentProto& document, std::string_view property_name) { for (const PropertyProto& property : document.properties()) { diff --git a/icing/schema/property-util.h b/icing/schema/property-util.h index efa599c..7557879 100644 --- a/icing/schema/property-util.h +++ b/icing/schema/property-util.h @@ -113,6 +113,26 @@ PropertyInfo ParsePropertyNameExpr(std::string_view property_name_expr); std::vector<PropertyInfo> ParsePropertyPathExpr( std::string_view property_path_expr); +// A property path property_path_expr1 is considered a parent of another +// property path property_path_expr2 if: +// 1. property_path_expr2 == property_path_expr1, OR +// 2. property_path_expr2 consists of the entire path of property_path_expr1 +// + "." + [some other property path]. +// +// Note that this can only be used for property name strings that do not +// contain the property index. +// +// Examples: +// - IsParentPropertyPath("foo", "foo") will return true. +// - IsParentPropertyPath("foo", "foo.bar") will return true. +// - IsParentPropertyPath("foo", "bar.foo") will return false. +// - IsParentPropertyPath("foo.bar", "foo.foo.bar") will return false. +// +// Returns: true if property_path_expr1 is a parent property path of +// property_path_expr2. +bool IsParentPropertyPath(std::string_view property_path_expr1, + std::string_view property_path_expr2); + // Gets the desired PropertyProto from the document by given property name. // Since the input parameter is property name, this function only deals with // the first level of properties in the document and cannot deal with nested diff --git a/icing/schema/property-util_test.cc b/icing/schema/property-util_test.cc index 1fabb32..eddcc84 100644 --- a/icing/schema/property-util_test.cc +++ b/icing/schema/property-util_test.cc @@ -43,6 +43,23 @@ static constexpr std::string_view kTypeNestedTest = "NestedTest"; static constexpr std::string_view kPropertyStr = "str"; static constexpr std::string_view kPropertyNestedDocument = "nestedDocument"; +TEST(PropertyUtilTest, IsParentPropertyPath) { + EXPECT_TRUE(property_util::IsParentPropertyPath("foo", "foo")); + EXPECT_TRUE(property_util::IsParentPropertyPath("foo", "foo.bar")); + EXPECT_TRUE(property_util::IsParentPropertyPath("foo", "foo.bar.foo")); + EXPECT_TRUE(property_util::IsParentPropertyPath("foo", "foo.foo.bar")); + EXPECT_TRUE(property_util::IsParentPropertyPath("foo.bar", "foo.bar.foo")); + + EXPECT_FALSE(property_util::IsParentPropertyPath("foo", "foofoo.bar")); + EXPECT_FALSE(property_util::IsParentPropertyPath("foo.bar", "foo.foo.bar")); + EXPECT_FALSE(property_util::IsParentPropertyPath("foo.bar", "foofoo.bar")); + EXPECT_FALSE(property_util::IsParentPropertyPath("foo.bar.foo", "foo")); + EXPECT_FALSE(property_util::IsParentPropertyPath("foo.bar.foo", "foo.bar")); + EXPECT_FALSE( + property_util::IsParentPropertyPath("foo.foo.bar", "foo.bar.foo")); + EXPECT_FALSE(property_util::IsParentPropertyPath("foo", "foo#bar.foo")); +} + TEST(PropertyUtilTest, ExtractPropertyValuesTypeString) { PropertyProto property; property.mutable_string_values()->Add("Hello, world"); diff --git a/icing/schema/schema-property-iterator.cc b/icing/schema/schema-property-iterator.cc index e1078c2..8fc245c 100644 --- a/icing/schema/schema-property-iterator.cc +++ b/icing/schema/schema-property-iterator.cc @@ -14,9 +14,17 @@ #include "icing/schema/schema-property-iterator.h" +#include <algorithm> +#include <string> +#include <unordered_set> +#include <utility> +#include <vector> + #include "icing/text_classifier/lib3/utils/base/status.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/absl_ports/str_cat.h" +#include "icing/proto/schema.pb.h" +#include "icing/schema/property-util.h" namespace icing { namespace lib { @@ -27,16 +35,63 @@ libtextclassifier3::Status SchemaPropertyIterator::Advance() { // When finishing iterating all properties of the current level, pop it // from the stack (levels_), return to the previous level and resume the // iteration. - parent_type_config_names_.erase(levels_.back().GetSchemaTypeName()); + parent_type_config_names_.erase( + parent_type_config_names_.find(levels_.back().GetSchemaTypeName())); levels_.pop_back(); continue; } const PropertyConfigProto& curr_property_config = levels_.back().GetCurrentPropertyConfig(); + std::string curr_property_path = levels_.back().GetCurrentPropertyPath(); + + // Iterate through the sorted_top_level_indexable_nested_properties_ in + // order until we find the first element that is >= curr_property_path. + while (current_top_level_indexable_nested_properties_idx_ < + sorted_top_level_indexable_nested_properties_.size() && + sorted_top_level_indexable_nested_properties_.at( + current_top_level_indexable_nested_properties_idx_) < + curr_property_path) { + // If an element in sorted_top_level_indexable_nested_properties_ < the + // current property path, it means that we've already iterated past the + // possible position for it without seeing it. + // It's not a valid property path in our schema definition. Add it to + // unknown_indexable_nested_properties_ and advance + // current_top_level_indexable_nested_properties_idx_. + unknown_indexable_nested_property_paths_.push_back( + sorted_top_level_indexable_nested_properties_.at( + current_top_level_indexable_nested_properties_idx_)); + ++current_top_level_indexable_nested_properties_idx_; + } + if (curr_property_config.data_type() != PropertyConfigProto::DataType::DOCUMENT) { // We've advanced to a leaf property. + // Set whether this property is indexable according to its level's + // indexable config. If this property is declared in + // indexable_nested_properties_list of the top-level schema, it is also + // nested indexable. + std::string* current_indexable_nested_prop = + current_top_level_indexable_nested_properties_idx_ < + sorted_top_level_indexable_nested_properties_.size() + ? &sorted_top_level_indexable_nested_properties_.at( + current_top_level_indexable_nested_properties_idx_) + : nullptr; + if (current_indexable_nested_prop == nullptr || + *current_indexable_nested_prop > curr_property_path) { + // Current property is not in the indexable list. Set it as indexable if + // its schema level is indexable AND it is an indexable property. + bool is_property_indexable = + levels_.back().GetLevelNestedIndexable() && + SchemaUtil::IsIndexedProperty(curr_property_config); + levels_.back().SetCurrentPropertyIndexable(is_property_indexable); + } else if (*current_indexable_nested_prop == curr_property_path) { + // Current property is in the indexable list. Set its indexable config + // to true. This property will consume a sectionId regardless of whether + // or not it is actually indexable. + levels_.back().SetCurrentPropertyIndexable(true); + ++current_top_level_indexable_nested_properties_idx_; + } return libtextclassifier3::Status::OK; } @@ -55,28 +110,87 @@ libtextclassifier3::Status SchemaPropertyIterator::Advance() { return absl_ports::NotFoundError(absl_ports::StrCat( "Type config not found: ", curr_property_config.schema_type())); } + const SchemaTypeConfigProto& nested_type_config = + nested_type_config_iter->second; - if (parent_type_config_names_.count( - nested_type_config_iter->second.schema_type()) > 0) { + if (levels_.back().GetLevelNestedIndexable()) { + // We should set sorted_top_level_indexable_nested_properties_ to the list + // defined by the current level. + // GetLevelNestedIndexable() is true either because: + // 1. We're looking at a document property of the top-level schema -- + // The first LevelInfo for the iterator is initialized with + // all_nested_properties_indexable_ = true. + // 2. All previous levels set index_nested_properties = true: + // This indicates that upper-level schema types want to follow nested + // properties definition of its document subtypes. If this is the first + // subtype level that defines a list, we should set it as + // top_level_indexable_nested_properties_ for the current top-level + // schema. + sorted_top_level_indexable_nested_properties_.clear(); + sorted_top_level_indexable_nested_properties_.reserve( + curr_property_config.document_indexing_config() + .indexable_nested_properties_list() + .size()); + for (const std::string& property : + curr_property_config.document_indexing_config() + .indexable_nested_properties_list()) { + // Concat the current property name to each property to get the full + // property path expression for each indexable nested property. + sorted_top_level_indexable_nested_properties_.push_back( + property_util::ConcatenatePropertyPathExpr(curr_property_path, + property)); + } + current_top_level_indexable_nested_properties_idx_ = 0; + // Sort elements and dedupe + std::sort(sorted_top_level_indexable_nested_properties_.begin(), + sorted_top_level_indexable_nested_properties_.end()); + auto last = + std::unique(sorted_top_level_indexable_nested_properties_.begin(), + sorted_top_level_indexable_nested_properties_.end()); + sorted_top_level_indexable_nested_properties_.erase( + last, sorted_top_level_indexable_nested_properties_.end()); + } + + bool is_cycle = + parent_type_config_names_.find(nested_type_config.schema_type()) != + parent_type_config_names_.end(); + bool is_parent_property_path = + current_top_level_indexable_nested_properties_idx_ < + sorted_top_level_indexable_nested_properties_.size() && + property_util::IsParentPropertyPath( + curr_property_path, + sorted_top_level_indexable_nested_properties_.at( + current_top_level_indexable_nested_properties_idx_)); + if (is_cycle && !is_parent_property_path) { // Cycle detected. The schema definition is guaranteed to be valid here // since it must have already been validated during SchemaUtil::Validate, // which would have rejected any schema with bad cycles. // + // There are no properties in the indexable_nested_properties_list that + // are a part of this circular reference. // We do not need to iterate this type further so we simply move on to // other properties in the parent type. continue; } - std::string curr_property_path = levels_.back().GetCurrentPropertyPath(); - bool is_nested_indexable = levels_.back().GetCurrentNestedIndexable() && - curr_property_config.document_indexing_config() - .index_nested_properties(); - levels_.push_back(LevelInfo(nested_type_config_iter->second, + bool all_nested_properties_indexable = + levels_.back().GetLevelNestedIndexable() && + curr_property_config.document_indexing_config() + .index_nested_properties(); + levels_.push_back(LevelInfo(nested_type_config, std::move(curr_property_path), - is_nested_indexable)); - parent_type_config_names_.insert( - nested_type_config_iter->second.schema_type()); + all_nested_properties_indexable)); + parent_type_config_names_.insert(nested_type_config.schema_type()); } + + // Before returning, move all remaining uniterated properties from + // sorted_top_level_indexable_nested_properties_ into + // unknown_indexable_nested_properties_. + std::move(sorted_top_level_indexable_nested_properties_.begin() + + current_top_level_indexable_nested_properties_idx_, + sorted_top_level_indexable_nested_properties_.end(), + std::back_inserter(unknown_indexable_nested_property_paths_)); + return absl_ports::OutOfRangeError("End of iterator"); } diff --git a/icing/schema/schema-property-iterator.h b/icing/schema/schema-property-iterator.h index f60a56e..66b8f32 100644 --- a/icing/schema/schema-property-iterator.h +++ b/icing/schema/schema-property-iterator.h @@ -18,6 +18,9 @@ #include <algorithm> #include <numeric> #include <string> +#include <string_view> +#include <unordered_set> +#include <utility> #include <vector> #include "icing/text_classifier/lib3/utils/base/status.h" @@ -44,7 +47,7 @@ class SchemaPropertyIterator { : type_config_map_(type_config_map) { levels_.push_back(LevelInfo(base_schema_type_config, /*base_property_path=*/"", - /*is_nested_indexable=*/true)); + /*all_nested_properties_indexable=*/true)); parent_type_config_names_.insert(base_schema_type_config.schema_type()); } @@ -62,11 +65,31 @@ class SchemaPropertyIterator { return levels_.back().GetCurrentPropertyPath(); } - // Gets if the current property is nested indexable. + // Returns whether the current property is indexable. This would be true if + // either the current level is nested indexable, or if the current property is + // declared indexable in the indexable_nested_properties_list of the top-level + // schema type. // // REQUIRES: The preceding call for Advance() is OK. - bool GetCurrentNestedIndexable() const { - return levels_.back().GetCurrentNestedIndexable(); + bool GetCurrentPropertyIndexable() const { + return levels_.back().GetCurrentPropertyIndexable(); + } + + // Returns whether the current schema level is nested indexable. If this is + // true, all properties in the level are indexed. + // + // REQUIRES: The preceding call for Advance() is OK. + bool GetLevelNestedIndexable() const { + return levels_.back().GetLevelNestedIndexable(); + } + + // The set of indexable nested properties that are defined in the + // indexable_nested_properties_list but are not found in the schema + // definition. These properties still consume sectionIds, but will not be + // indexed. + const std::vector<std::string>& unknown_indexable_nested_property_paths() + const { + return unknown_indexable_nested_property_paths_; } // Advances to the next leaf property. @@ -87,12 +110,14 @@ class SchemaPropertyIterator { class LevelInfo { public: explicit LevelInfo(const SchemaTypeConfigProto& schema_type_config, - std::string base_property_path, bool is_nested_indexable) + std::string base_property_path, + bool all_nested_properties_indexable) : schema_type_config_(schema_type_config), base_property_path_(std::move(base_property_path)), sorted_property_indices_(schema_type_config.properties_size()), current_vec_idx_(-1), - is_nested_indexable_(is_nested_indexable) { + sorted_property_indexable_(schema_type_config.properties_size()), + all_nested_properties_indexable_(all_nested_properties_indexable) { // Index sort property by lexicographical order. std::iota(sorted_property_indices_.begin(), sorted_property_indices_.end(), @@ -119,7 +144,17 @@ class SchemaPropertyIterator { base_property_path_, GetCurrentPropertyConfig().property_name()); } - bool GetCurrentNestedIndexable() const { return is_nested_indexable_; } + bool GetLevelNestedIndexable() const { + return all_nested_properties_indexable_; + } + + bool GetCurrentPropertyIndexable() const { + return sorted_property_indexable_[current_vec_idx_]; + } + + void SetCurrentPropertyIndexable(bool indexable) { + sorted_property_indexable_[current_vec_idx_] = indexable; + } std::string_view GetSchemaTypeName() const { return schema_type_config_.schema_type(); @@ -137,12 +172,20 @@ class SchemaPropertyIterator { std::vector<int> sorted_property_indices_; int current_vec_idx_; - // Indicates if the current level is nested indexable. Document type - // property has index_nested_properties flag indicating whether properties - // under this level should be indexed or not. If any of parent document type - // property sets its flag false, then all child level properties should not - // be indexed. - bool is_nested_indexable_; + // Vector indicating whether each property in the current level is + // indexable. We can declare different indexable settings for properties in + // the same level using indexable_nested_properties_list. + // + // Element indices in this vector correspond to property indices in the + // sorted order. + std::vector<bool> sorted_property_indexable_; + + // Indicates if all properties in the current level is nested indexable. + // This would be true for a level if the document declares + // index_nested_properties=true. If any of parent document type + // property sets its flag false, then this would be false for all its child + // properties. + bool all_nested_properties_indexable_; }; const SchemaUtil::TypeConfigMap& type_config_map_; // Does not own @@ -154,7 +197,23 @@ class SchemaPropertyIterator { // Maintaining all traversed parent schema type config names of the current // stack (levels_). It is used to detect nested schema cycle dependency. - std::unordered_set<std::string_view> parent_type_config_names_; + std::unordered_multiset<std::string_view> parent_type_config_names_; + + // Sorted list of indexable nested properties for the top-level schema. + std::vector<std::string> sorted_top_level_indexable_nested_properties_; + + // Current iteration index in the sorted_top_level_indexable_nested_properties + // list. + int current_top_level_indexable_nested_properties_idx_ = 0; + + // Vector of indexable nested properties defined in the + // indexable_nested_properties_list, but not found in the schema definition. + // These properties still consume sectionIds, but will not be indexed. + // Properties are inserted into this vector in sorted order. + // + // TODO(b/289152024): Implement support for indexing these properties if they + // are in the child types of polymorphic nested properties. + std::vector<std::string> unknown_indexable_nested_property_paths_; }; } // namespace lib diff --git a/icing/schema/schema-property-iterator_test.cc b/icing/schema/schema-property-iterator_test.cc index 080d574..2b0226d 100644 --- a/icing/schema/schema-property-iterator_test.cc +++ b/icing/schema/schema-property-iterator_test.cc @@ -14,6 +14,7 @@ #include "icing/schema/schema-property-iterator.h" +#include <initializer_list> #include <string> #include "icing/text_classifier/lib3/utils/base/status.h" @@ -30,7 +31,9 @@ namespace lib { namespace { using portable_equals_proto::EqualsProto; +using ::testing::ElementsAre; using ::testing::Eq; +using ::testing::IsEmpty; using ::testing::IsFalse; using ::testing::IsTrue; @@ -41,13 +44,14 @@ TEST(SchemaPropertyIteratorTest, SchemaTypeConfigProto schema_type_config = SchemaTypeConfigBuilder() .SetType(schema_type_name) - .AddProperty(PropertyConfigBuilder().SetName("Google").SetDataType( - TYPE_STRING)) + .AddProperty( + PropertyConfigBuilder().SetName("Google").SetDataTypeString( + TERM_MATCH_EXACT, TOKENIZER_PLAIN)) .AddProperty(PropertyConfigBuilder().SetName("Youtube").SetDataType( TYPE_BYTES)) .AddProperty(PropertyConfigBuilder() .SetName("Alphabet") - .SetDataType(TYPE_INT64)) + .SetDataTypeInt64(NUMERIC_MATCH_UNKNOWN)) .Build(); SchemaUtil::TypeConfigMap type_config_map = { {schema_type_name, schema_type_config}}; @@ -57,22 +61,24 @@ TEST(SchemaPropertyIteratorTest, EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Alphabet")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config.properties(2))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Google")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config.properties(0))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Youtube")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config.properties(1))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(iterator.unknown_indexable_nested_property_paths(), IsEmpty()); } TEST(SchemaPropertyIteratorTest, @@ -84,19 +90,20 @@ TEST(SchemaPropertyIteratorTest, SchemaTypeConfigProto schema_type_config1 = SchemaTypeConfigBuilder() .SetType(schema_type_name1) - .AddProperty(PropertyConfigBuilder().SetName("Google").SetDataType( - TYPE_STRING)) + .AddProperty( + PropertyConfigBuilder().SetName("Google").SetDataTypeString( + TERM_MATCH_EXACT, TOKENIZER_PLAIN)) .AddProperty(PropertyConfigBuilder().SetName("Youtube").SetDataType( TYPE_BYTES)) .AddProperty(PropertyConfigBuilder() .SetName("Alphabet") - .SetDataType(TYPE_INT64)) + .SetDataTypeInt64(NUMERIC_MATCH_RANGE)) .Build(); SchemaTypeConfigProto schema_type_config2 = SchemaTypeConfigBuilder() .SetType(schema_type_name2) - .AddProperty( - PropertyConfigBuilder().SetName("Foo").SetDataType(TYPE_STRING)) + .AddProperty(PropertyConfigBuilder().SetName("Foo").SetDataTypeString( + TERM_MATCH_UNKNOWN, TOKENIZER_NONE)) .AddProperty( PropertyConfigBuilder().SetName("Bar").SetDataTypeDocument( schema_type_name1, /*index_nested_properties=*/true)) @@ -105,7 +112,8 @@ TEST(SchemaPropertyIteratorTest, SchemaTypeConfigBuilder() .SetType(schema_type_name3) .AddProperty( - PropertyConfigBuilder().SetName("Hello").SetDataType(TYPE_STRING)) + PropertyConfigBuilder().SetName("Hello").SetDataTypeString( + TERM_MATCH_EXACT, TOKENIZER_PLAIN)) .AddProperty( PropertyConfigBuilder().SetName("World").SetDataTypeDocument( schema_type_name1, /*index_nested_properties=*/true)) @@ -139,52 +147,54 @@ TEST(SchemaPropertyIteratorTest, EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Hello")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config3.properties(0))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Icing.Bar.Alphabet")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(2))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Icing.Bar.Google")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(0))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Icing.Bar.Youtube")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(1))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Icing.Foo")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config2.properties(0))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("World.Alphabet")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(2))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("World.Google")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(0))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("World.Youtube")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(1))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(iterator.unknown_indexable_nested_property_paths(), IsEmpty()); } TEST(SchemaPropertyIteratorTest, @@ -234,6 +244,7 @@ TEST(SchemaPropertyIteratorTest, SchemaPropertyIterator iterator(schema_type_config, type_config_map); EXPECT_THAT(iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + EXPECT_THAT(iterator.unknown_indexable_nested_property_paths(), IsEmpty()); } TEST(SchemaPropertyIteratorTest, NestedIndexable) { @@ -338,13 +349,13 @@ TEST(SchemaPropertyIteratorTest, NestedIndexable) { EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Baz1.Bar.Google")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(0))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Baz1.Foo")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config2.properties(1))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsTrue()); // For Baz2, the parent level sets index_nested_properties = false, so all // leaf properties in child levels should be nested unindexable even if @@ -353,13 +364,13 @@ TEST(SchemaPropertyIteratorTest, NestedIndexable) { EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Baz2.Bar.Google")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(0))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Baz2.Foo")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config2.properties(1))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsFalse()); // For Baz3, the parent level sets index_nested_properties = true, but the // child level sets index_nested_properties = false. @@ -369,13 +380,13 @@ TEST(SchemaPropertyIteratorTest, NestedIndexable) { EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Baz3.Bar.Google")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(0))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Baz3.Foo")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config2.properties(1))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsTrue()); // For Baz4, all levels set index_nested_properties = false, so all leaf // properties should be nested unindexable. @@ -383,37 +394,1498 @@ TEST(SchemaPropertyIteratorTest, NestedIndexable) { EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Baz4.Bar.Google")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(0))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Baz4.Foo")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config2.properties(1))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsFalse()); // Verify 1 and 0 level of nested document type properties. EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Hello1.Google")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(0))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("Hello2.Google")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config1.properties(0))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(iterator.Advance(), IsOk()); EXPECT_THAT(iterator.GetCurrentPropertyPath(), Eq("World")); EXPECT_THAT(iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config4.properties(6))); - EXPECT_THAT(iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(iterator.unknown_indexable_nested_property_paths(), IsEmpty()); +} + +TEST(SchemaPropertyIteratorTest, + IndexableNestedPropertiesList_singleNestedLevel) { + std::string schema_type_name1 = "SchemaOne"; + std::string schema_type_name2 = "SchemaTwo"; + + SchemaTypeConfigProto schema_type_config1 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name1) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop1") + .SetDataTypeString(TERM_MATCH_UNKNOWN, TOKENIZER_NONE)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop2") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop3") + .SetDataTypeString(TERM_MATCH_UNKNOWN, TOKENIZER_NONE)) + .AddProperty(PropertyConfigBuilder() + .SetName("schema1prop4") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE)) + .AddProperty(PropertyConfigBuilder() + .SetName("schema1prop5") + .SetDataType(TYPE_BOOLEAN)) + .Build(); + SchemaTypeConfigProto schema_type_config2 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name2) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop1") + .SetDataTypeDocument( + schema_type_name1, + /*indexable_nested_properties_list=*/{"schema1prop2", + "schema1prop3", + "schema1prop5"})) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty(PropertyConfigBuilder() + .SetName("schema2prop3") + .SetDataTypeInt64(NUMERIC_MATCH_UNKNOWN)) + .Build(); + SchemaUtil::TypeConfigMap type_config_map = { + {schema_type_name1, schema_type_config1}, + {schema_type_name2, schema_type_config2}}; + + // Order of iteration for Schema2: + // {"schema2prop1.schema1prop1", "schema2prop1.schema1prop2", + // "schema2prop1.schema1prop3", "schema2prop1.schema1prop4", + // "schema2prop1.schema1prop5", "schema2prop2", "schema2prop3"} + // + // Indexable properties: + // {"schema2prop1.schema1prop2", "schema2prop1.schema1prop3", + // "schema2prop1.schema1prop5", "schema2prop2"}. + // + // "schema2prop1.schema1prop4" is indexable by its indexing-config, but is not + // considered indexable for Schema2 because Schema2 sets its + // index_nested_properties config to false, and "schema1prop4" is not + // in the indexable_nested_properties_list for schema2prop1. + // + // "schema2prop1.schema1prop1", "schema2prop1.schema1prop3" and + // "schema2prop1.schema1prop5" are non-indexable by its indexing-config. + // However "schema2prop1.schema1prop3" and "schema2prop1.schema1prop5" are + // indexed as it appears in the indexable_list. + SchemaPropertyIterator schema2_iterator(schema_type_config2, type_config_map); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop1")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop3")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop4")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(3))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop5")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(4))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), Eq("schema2prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), Eq("schema2prop3")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(2))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema2_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Iterate through schema1 properties. Schema1 only has non-document type leaf + // properties, so its properties will be assigned indexable or not according + // to their indexing configs. + SchemaPropertyIterator schema1_iterator(schema_type_config1, type_config_map); + + EXPECT_THAT(schema1_iterator.Advance(), IsOk()); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyPath(), Eq("schema1prop1")); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema1_iterator.Advance(), IsOk()); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyPath(), Eq("schema1prop2")); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema1_iterator.Advance(), IsOk()); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyPath(), Eq("schema1prop3")); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema1_iterator.Advance(), IsOk()); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyPath(), Eq("schema1prop4")); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(3))); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema1_iterator.Advance(), IsOk()); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyPath(), Eq("schema1prop5")); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(4))); + EXPECT_THAT(schema1_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema1_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema1_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); +} + +TEST(SchemaPropertyIteratorTest, + IndexableNestedPropertiesList_indexBooleanTrueDoesNotAffectOtherLevels) { + std::string schema_type_name1 = "SchemaOne"; + std::string schema_type_name2 = "SchemaTwo"; + std::string schema_type_name3 = "SchemaThree"; + + SchemaTypeConfigProto schema_type_config1 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name1) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop1") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop2") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop3") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config2 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name2) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop1") + .SetDataTypeDocument(schema_type_name1, + /*index_nested_properties=*/true)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop3") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config3 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name3) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema3prop3") + .SetDataTypeDocument( + schema_type_name1, + /*indexable_nested_properties_list=*/{"schema1prop1", + "schema1prop3"})) + .AddProperty(PropertyConfigBuilder() + .SetName("schema3prop1") + .SetDataTypeDocument( + schema_type_name2, + /*indexable_nested_properties_list=*/ + {"schema2prop2", "schema2prop1.schema1prop1", + "schema2prop1.schema1prop3"})) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema3prop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaUtil::TypeConfigMap type_config_map = { + {schema_type_name1, schema_type_config1}, + {schema_type_name2, schema_type_config2}, + {schema_type_name3, schema_type_config3}}; + + // Order of iteration for Schema3: + // {"schema3prop1.schema2prop1.schema1prop1", + // "schema3prop1.schema2prop1.schema1prop2", + // "schema3prop1.schema2prop1.schema1prop3", + // "schema3prop1.schema2prop2", "schema3prop1.schema2prop3", "schema3prop2", + // "schema3prop3.schema1prop1", "schema3prop3.schema1prop2", + // "schema3prop3.schema1prop3"}. + // + // Indexable properties: + // {"schema3prop1.schema2prop1.schema1prop1", + // "schema3prop1.schema2prop1.schema1prop3", + // "schema3prop1.schema2prop2", "schema3prop2", "schema3prop3.schema1prop1", + // "schema3prop3.schema1prop3"} + // + // Schema2 setting index_nested_properties=true does not affect nested + // properties indexing for Schema3. + SchemaPropertyIterator schema3_iterator(schema_type_config3, type_config_map); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop3")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop3")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), Eq("schema3prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config3.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop3.schema1prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop3.schema1prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop3.schema1prop3")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema3_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration for Schema2: + // {"schema2prop1.schema1prop1", "schema2prop1.schema1prop2", + // "schema2prop1.schema1prop3", "schema2prop2", "schema2prop3"} + // + // Indexable properties: + // {"schema2prop1.schema1prop1", "schema2prop1.schema1prop2", + // "schema2prop1.schema1prop3", "schema2prop2", "schema2prop3"} + // + // All properties are indexed because index_nested_properties=true for + // Schema2.schema2prop1. Schema3's indexable_nested_properties setting does + // not affect this. + SchemaPropertyIterator schema2_iterator(schema_type_config2, type_config_map); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop1")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop3")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), Eq("schema2prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), Eq("schema2prop3")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(2))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema2_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); +} + +TEST(SchemaPropertyIteratorTest, + IndexableNestedPropertiesList_indexBooleanFalseDoesNotAffectOtherLevels) { + std::string schema_type_name1 = "SchemaOne"; + std::string schema_type_name2 = "SchemaTwo"; + std::string schema_type_name3 = "SchemaThree"; + + SchemaTypeConfigProto schema_type_config1 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name1) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop1") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop2") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config2 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name2) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop1") + .SetDataTypeDocument(schema_type_name1, + /*index_nested_properties=*/false)) + .Build(); + SchemaTypeConfigProto schema_type_config3 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name3) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema3prop1") + .SetDataTypeDocument(schema_type_name2, + /*indexable_nested_properties_list=*/ + std::initializer_list<std::string>{ + "schema2prop1.schema1prop2"})) + .Build(); + SchemaUtil::TypeConfigMap type_config_map = { + {schema_type_name1, schema_type_config1}, + {schema_type_name2, schema_type_config2}, + {schema_type_name3, schema_type_config3}}; + + // Order of iteration for Schema3: + // {"schema3prop1.schema2prop1.schema1prop1", + // "schema3prop1.schema2prop1.schema1prop2"}. + // + // Indexable properties: {"schema3prop1.schema2prop1.schema1prop2"} + // + // Schema2 setting index_nested_properties=false, does not affect Schema3's + // indexable list. + SchemaPropertyIterator schema3_iterator(schema_type_config3, type_config_map); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema3_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration for Schema2: + // {"schema2prop1.schema1prop1", "schema2prop1.schema1prop2"} + // + // Indexable properties: None + // + // The indexable list for Schema3 does not propagate to Schema2. + SchemaPropertyIterator schema2_iterator(schema_type_config2, type_config_map); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop1")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema2_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); +} + +TEST(SchemaPropertyIteratorTest, + IndexableNestedPropertiesList_indexableSetDoesNotAffectOtherLevels) { + std::string schema_type_name1 = "SchemaOne"; + std::string schema_type_name2 = "SchemaTwo"; + std::string schema_type_name3 = "SchemaThree"; + + SchemaTypeConfigProto schema_type_config1 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name1) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop1") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop2") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop3") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config2 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name2) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop1") + .SetDataTypeDocument( + schema_type_name1, + /*indexable_nested_properties_list=*/ + std::initializer_list<std::string>{"schema1prop2"})) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop3") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config3 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name3) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema3prop3") + .SetDataTypeDocument( + schema_type_name1, + /*indexable_nested_properties_list=*/{"schema1prop1", + "schema1prop3"})) + .AddProperty(PropertyConfigBuilder() + .SetName("schema3prop1") + .SetDataTypeDocument( + schema_type_name2, + /*indexable_nested_properties_list=*/ + {"schema2prop2", "schema2prop1.schema1prop1", + "schema2prop1.schema1prop3"})) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema3prop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaUtil::TypeConfigMap type_config_map = { + {schema_type_name1, schema_type_config1}, + {schema_type_name2, schema_type_config2}, + {schema_type_name3, schema_type_config3}}; + + // Order of iteration for Schema3: + // {"schema3prop1.schema2prop1.schema1prop1", + // "schema3prop1.schema2prop1.schema1prop2", + // "schema3prop1.schema2prop1.schema1prop3", + // "schema3prop1.schema2prop2", "schema3prop1.schema2prop3", "schema3prop2", + // "schema3prop3.schema1prop1", "schema3prop3.schema1prop2", + // "schema3prop3.schema1prop3"}. + // + // Indexable properties: + // {"schema3prop1.schema2prop1.schema1prop1", + // "schema3prop1.schema2prop1.schema1prop3", + // "schema3prop1.schema2prop2", "schema3prop2", "schema3prop3.schema1prop1", + // "schema3prop3.schema1prop3"} + // + // Schema2 setting indexable_nested_properties_list={schema1prop2} does not + // affect nested properties indexing for Schema3. + SchemaPropertyIterator schema3_iterator(schema_type_config3, type_config_map); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop3")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop3")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), Eq("schema3prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config3.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop3.schema1prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop3.schema1prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop3.schema1prop3")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema3_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration for Schema2: + // {"schema2prop1.schema1prop1", "schema2prop1.schema1prop2", + // "schema2prop1.schema1prop3", "schema2prop2", "schema2prop3"} + // + // Indexable properties: + // {"schema2prop1.schema1prop2", "schema2prop2", "schema2prop3"} + // + // Indexable_nested_properties set for Schema3.schema3prop1 does not propagate + // to Schema2. + SchemaPropertyIterator schema2_iterator(schema_type_config2, type_config_map); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop1")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop3")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), Eq("schema2prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), Eq("schema2prop3")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(2))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema2_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); +} + +TEST( + SchemaPropertyIteratorTest, + IndexableNestedPropertiesList_upperLevelIndexTrueIndexesListOfNestedLevel) { + std::string schema_type_name1 = "SchemaOne"; + std::string schema_type_name2 = "SchemaTwo"; + std::string schema_type_name3 = "SchemaThree"; + std::string schema_type_name4 = "SchemaFour"; + + SchemaTypeConfigProto schema_type_config1 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name1) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop1") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop2") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config2 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name2) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop1") + .SetDataTypeDocument( + schema_type_name1, + /*indexable_nested_properties_list=*/ + std::initializer_list<std::string>{"schema1prop2"})) + .Build(); + SchemaTypeConfigProto schema_type_config3 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name3) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema3prop1") + .SetDataTypeDocument(schema_type_name2, + /*index_nested_properties=*/true)) + .Build(); + SchemaTypeConfigProto schema_type_config4 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name4) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema4prop1") + .SetDataTypeDocument(schema_type_name3, + /*index_nested_properties=*/true)) + .Build(); + SchemaUtil::TypeConfigMap type_config_map = { + {schema_type_name1, schema_type_config1}, + {schema_type_name2, schema_type_config2}, + {schema_type_name3, schema_type_config3}, + {schema_type_name4, schema_type_config4}}; + + // Order of iteration for Schema4: + // {"schema4prop1.schema3prop1.schema2prop1.schema1prop1", + // "schema4prop1.schema3prop1.schema2prop1.schema1prop2"}. + // + // Indexable properties: {schema4prop1.schema3prop1.schema2prop1.schema1prop2} + // + // Both Schema4 and Schema3 sets index_nested_properties=true, so they both + // want to follow the indexing behavior of its subtype. + // Schema2 is the first subtype to define an indexing config, so we index its + // list for both Schema3 and Schema4 even though it sets + // index_nested_properties=false. + SchemaPropertyIterator schema4_iterator(schema_type_config4, type_config_map); + + EXPECT_THAT(schema4_iterator.Advance(), IsOk()); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyPath(), + Eq("schema4prop1.schema3prop1.schema2prop1.schema1prop1")); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema4_iterator.Advance(), IsOk()); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyPath(), + Eq("schema4prop1.schema3prop1.schema2prop1.schema1prop2")); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema4_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema4_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration for Schema3: + // {"schema3prop1.schema2prop1.schema1prop1", + // "schema3prop1.schema2prop1.schema1prop2"}. + // + // Indexable properties: {schema3prop1.schema2prop1.schema1prop2} + SchemaPropertyIterator schema3_iterator(schema_type_config3, type_config_map); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema3_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration for Schema2: + // {"schema2prop1.schema1prop1", "schema2prop1.schema1prop2"} + // + // Indexable properties: + // {"schema2prop1.schema1prop2"} + // + // Schema3 setting index_nested_properties=true does not propagate to Schema2. + SchemaPropertyIterator schema2_iterator(schema_type_config2, type_config_map); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop1")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema2_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); +} + +TEST(SchemaPropertyIteratorTest, + IndexableNestedPropertiesList_unknownPropPaths) { + std::string schema_type_name1 = "SchemaOne"; + std::string schema_type_name2 = "SchemaTwo"; + std::string schema_type_name3 = "SchemaThree"; + std::string schema_type_name4 = "SchemaFour"; + + SchemaTypeConfigProto schema_type_config1 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name1) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop1") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop2") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config2 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name2) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop1") + .SetDataTypeDocument(schema_type_name1, + /*indexable_nested_properties_list=*/ + {"schema1prop2", "schema1prop2.foo", + "foo.bar", "zzz", "aaa.zzz"})) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop2") + .SetDataTypeDocument( + schema_type_name1, + /*indexable_nested_properties_list=*/ + {"schema1prop1", "schema1prop2", "unknown.path"})) + .Build(); + SchemaTypeConfigProto schema_type_config3 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name3) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema3prop1") + .SetDataTypeDocument( + schema_type_name2, + /*indexable_nested_properties_list=*/ + {"schema3prop1", "schema2prop1", "schema1prop2", + "schema2prop1.schema1prop2", "schema2prop1.zzz", "zzz"})) + .Build(); + SchemaTypeConfigProto schema_type_config4 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name4) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema4prop1") + .SetDataTypeDocument(schema_type_name3, + /*index_nested_properties=*/true)) + .Build(); + SchemaUtil::TypeConfigMap type_config_map = { + {schema_type_name1, schema_type_config1}, + {schema_type_name2, schema_type_config2}, + {schema_type_name3, schema_type_config3}, + {schema_type_name4, schema_type_config4}}; + + // Order of iteration for Schema4: + // "schema4prop1.schema3prop1.schema2prop1.schema1prop1", + // "schema4prop1.schema3prop1.schema2prop1.schema1prop2" (indexable), + // "schema4prop1.schema3prop1.schema2prop2.schema1prop1", + // "schema4prop1.schema3prop1.schema2prop2.schema1prop2" + // + // Unknown property paths from schema3 will also be included for schema4, + // since schema4 sets index_nested_properties=true. + // This includes everything in schema3prop1's list except + // "schema2prop1.schema1prop2". + SchemaPropertyIterator schema4_iterator(schema_type_config4, type_config_map); + + EXPECT_THAT(schema4_iterator.Advance(), IsOk()); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyPath(), + Eq("schema4prop1.schema3prop1.schema2prop1.schema1prop1")); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema4_iterator.Advance(), IsOk()); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyPath(), + Eq("schema4prop1.schema3prop1.schema2prop1.schema1prop2")); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema4_iterator.Advance(), IsOk()); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyPath(), + Eq("schema4prop1.schema3prop1.schema2prop2.schema1prop1")); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema4_iterator.Advance(), IsOk()); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyPath(), + Eq("schema4prop1.schema3prop1.schema2prop2.schema1prop2")); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema4_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema4_iterator.unknown_indexable_nested_property_paths(), + testing::ElementsAre("schema4prop1.schema3prop1.schema1prop2", + "schema4prop1.schema3prop1.schema2prop1", + "schema4prop1.schema3prop1.schema2prop1.zzz", + "schema4prop1.schema3prop1.schema3prop1", + "schema4prop1.schema3prop1.zzz")); + + // Order of iteration for Schema3: + // "schema3prop1.schema2prop1.schema1prop1", + // "schema3prop1.schema2prop1.schema1prop2" (indexable), + // "schema3prop1.schema2prop2.schema1prop1", + // "schema3prop1.schema2prop2.schema1prop2" + // + // Unknown properties (in order): + // "schema3prop1.schema1prop2", "schema3prop1.schema2prop1" (not a leaf prop), + // "schema3prop1.schema2prop1.zzz", "schema3prop1.schema3prop1", + // "schema3prop1.zzz" + SchemaPropertyIterator schema3_iterator(schema_type_config3, type_config_map); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop2.schema1prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop2.schema1prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema3_iterator.unknown_indexable_nested_property_paths(), + testing::ElementsAre( + "schema3prop1.schema1prop2", "schema3prop1.schema2prop1", + "schema3prop1.schema2prop1.zzz", "schema3prop1.schema3prop1", + "schema3prop1.zzz")); + + // Order of iteration for Schema2: + // "schema2prop1.schema1prop1", + // "schema2prop1.schema1prop2" (indexable), + // "schema2prop2.schema1prop1" (indexable), + // "schema2prop2.schema1prop2" (indexable) + // + // Unknown properties (in order): + // "schema2prop1.aaa.zzz", "schema2prop1.foo.bar", + // "schema2prop1.schema1prop2.foo", "schema2prop1.zzz", + // "schema2prop2.unknown.path" + SchemaPropertyIterator schema2_iterator(schema_type_config2, type_config_map); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop1")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop2.schema1prop1")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop2.schema1prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT( + schema2_iterator.unknown_indexable_nested_property_paths(), + testing::ElementsAre("schema2prop1.aaa.zzz", "schema2prop1.foo.bar", + "schema2prop1.schema1prop2.foo", "schema2prop1.zzz", + "schema2prop2.unknown.path")); +} + +TEST(SchemaPropertyIteratorTest, + IndexableNestedPropertiesListDuplicateElements) { + std::string schema_type_name1 = "SchemaOne"; + std::string schema_type_name2 = "SchemaTwo"; + std::string schema_type_name3 = "SchemaThree"; + std::string schema_type_name4 = "SchemaFour"; + + SchemaTypeConfigProto schema_type_config1 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name1) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop1") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema1prop2") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config2 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name2) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema2prop1") + .SetDataTypeDocument( + schema_type_name1, + /*indexable_nested_properties_list=*/ + {"schema1prop2", "schema1prop2", "schema1prop2.foo", + "schema1prop2.foo", "foo.bar", "foo.bar", "foo.bar", + "zzz", "zzz", "aaa.zzz", "schema1prop2"})) + .AddProperty(PropertyConfigBuilder() + .SetName("schema2prop2") + .SetDataTypeDocument( + schema_type_name1, + /*indexable_nested_properties_list=*/ + {"schema1prop1", "schema1prop2", "unknown.path", + "unknown.path", "unknown.path", "unknown.path", + "schema1prop1"})) + .Build(); + SchemaTypeConfigProto schema_type_config3 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name3) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema3prop1") + .SetDataTypeDocument( + schema_type_name2, + /*indexable_nested_properties_list=*/ + {"schema3prop1", "schema3prop1", "schema2prop1", + "schema2prop1", "schema1prop2", "schema1prop2", + "schema2prop1.schema1prop2", "schema2prop1.schema1prop2", + "schema2prop1.zzz", "zzz", "zzz"})) + .Build(); + SchemaTypeConfigProto schema_type_config4 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name4) + .AddProperty( + PropertyConfigBuilder() + .SetName("schema4prop1") + .SetDataTypeDocument(schema_type_name3, + /*index_nested_properties=*/true)) + .Build(); + SchemaUtil::TypeConfigMap type_config_map = { + {schema_type_name1, schema_type_config1}, + {schema_type_name2, schema_type_config2}, + {schema_type_name3, schema_type_config3}, + {schema_type_name4, schema_type_config4}}; + + // The results of this test case is the same as the previous test case. This + // is to test that the indexable-list is deduped correctly. + + // Order of iteration for Schema4: + // "schema4prop1.schema3prop1.schema2prop1.schema1prop1", + // "schema4prop1.schema3prop1.schema2prop1.schema1prop2" (indexable), + // "schema4prop1.schema3prop1.schema2prop2.schema1prop1", + // "schema4prop1.schema3prop1.schema2prop2.schema1prop2" + // + // Unknown property paths from schema3 will also be included for schema4, + // since schema4 sets index_nested_properties=true. + // This includes everything in schema3prop1's list except + // "schema2prop1.schema1prop2". + SchemaPropertyIterator schema4_iterator(schema_type_config4, type_config_map); + + EXPECT_THAT(schema4_iterator.Advance(), IsOk()); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyPath(), + Eq("schema4prop1.schema3prop1.schema2prop1.schema1prop1")); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema4_iterator.Advance(), IsOk()); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyPath(), + Eq("schema4prop1.schema3prop1.schema2prop1.schema1prop2")); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema4_iterator.Advance(), IsOk()); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyPath(), + Eq("schema4prop1.schema3prop1.schema2prop2.schema1prop1")); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema4_iterator.Advance(), IsOk()); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyPath(), + Eq("schema4prop1.schema3prop1.schema2prop2.schema1prop2")); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema4_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema4_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema4_iterator.unknown_indexable_nested_property_paths(), + testing::ElementsAre("schema4prop1.schema3prop1.schema1prop2", + "schema4prop1.schema3prop1.schema2prop1", + "schema4prop1.schema3prop1.schema2prop1.zzz", + "schema4prop1.schema3prop1.schema3prop1", + "schema4prop1.schema3prop1.zzz")); + + // Order of iteration for Schema3: + // "schema3prop1.schema2prop1.schema1prop1", + // "schema3prop1.schema2prop1.schema1prop2" (indexable), + // "schema3prop1.schema2prop2.schema1prop1", + // "schema3prop1.schema2prop2.schema1prop2" + // + // Unknown properties (in order): + // "schema2prop1.aaa.zzz", "schema2prop1.foo.bar", + // "schema2prop1.schema1prop2.foo", "schema2prop1.zzz", + // "schema2prop2.unknown.path" + SchemaPropertyIterator schema3_iterator(schema_type_config3, type_config_map); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop1.schema1prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop2.schema1prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("schema3prop1.schema2prop2.schema1prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema3_iterator.unknown_indexable_nested_property_paths(), + testing::ElementsAre( + "schema3prop1.schema1prop2", "schema3prop1.schema2prop1", + "schema3prop1.schema2prop1.zzz", "schema3prop1.schema3prop1", + "schema3prop1.zzz")); + + // Order of iteration for Schema2: + // "schema2prop1.schema1prop1", + // "schema2prop1.schema1prop2" (indexable), + // "schema2prop2.schema1prop1" (indexable), + // "schema2prop2.schema1prop2" (indexable) + // + // Unknown properties (in order): + // "schema2prop1.aaa.zzz", "schema2prop1.foo.bar", + // "schema2prop1.schema1prop2.foo", "schema2prop1.zzz", + // "schema2prop2.unknown.path" + SchemaPropertyIterator schema2_iterator(schema_type_config2, type_config_map); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop1")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop1.schema1prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop2.schema1prop1")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), + Eq("schema2prop2.schema1prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT( + schema2_iterator.unknown_indexable_nested_property_paths(), + testing::ElementsAre("schema2prop1.aaa.zzz", "schema2prop1.foo.bar", + "schema2prop1.schema1prop2.foo", "schema2prop1.zzz", + "schema2prop2.unknown.path")); } +TEST(SchemaPropertyIteratorTest, + IndexableNestedProperties_duplicatePropertyNamesInDifferentProperties) { + std::string schema_type_name1 = "SchemaOne"; + std::string schema_type_name2 = "SchemaTwo"; + std::string schema_type_name3 = "SchemaThree"; + + SchemaTypeConfigProto schema_type_config1 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name1) + .AddProperty( + PropertyConfigBuilder().SetName("prop1").SetDataTypeString( + TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder().SetName("prop2").SetDataTypeString( + TERM_MATCH_PREFIX, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder().SetName("prop3").SetDataTypeString( + TERM_MATCH_PREFIX, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config2 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name2) + .AddProperty( + PropertyConfigBuilder().SetName("prop1").SetDataTypeDocument( + schema_type_name1, + /*indexable_nested_properties_list=*/ + std::initializer_list<std::string>{"prop2"})) + .AddProperty( + PropertyConfigBuilder().SetName("prop2").SetDataTypeString( + TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder().SetName("prop3").SetDataTypeString( + TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config3 = + SchemaTypeConfigBuilder() + .SetType(schema_type_name3) + .AddProperty( + PropertyConfigBuilder().SetName("prop3").SetDataTypeDocument( + schema_type_name1, + /*indexable_nested_properties_list=*/ + {"prop1", "prop3"})) + .AddProperty( + PropertyConfigBuilder().SetName("prop1").SetDataTypeDocument( + schema_type_name2, + /*indexable_nested_properties_list=*/ + {"prop2", "prop1.prop1", "prop1.prop3"})) + .AddProperty( + PropertyConfigBuilder().SetName("prop2").SetDataTypeString( + TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder().SetName("prop4").SetDataTypeDocument( + schema_type_name1, + /*indexable_nested_properties_list=*/ + {"prop2", "prop3"})) + .Build(); + SchemaUtil::TypeConfigMap type_config_map = { + {schema_type_name1, schema_type_config1}, + {schema_type_name2, schema_type_config2}, + {schema_type_name3, schema_type_config3}}; + + // Order of iteration for Schema3: + // {"prop1.prop1.prop1", "prop1.prop1.prop2", "prop1.prop1.prop3", + // "prop1.prop2", "prop1.prop3", "prop2", + // "prop3.prop1", "prop3.prop2", "prop3.prop3", + // "prop4.prop1", "prop4.prop2", "prop4.prop3"}. + // + // Indexable properties: + // {"prop1.prop1.prop1", "prop1.prop1.prop3", "prop1.prop2", "prop2", + // "prop3.prop1", "prop3.prop3", "prop4.prop2", "prop4.prop3"} + // + // Properties do not affect other properties with the same name from different + // properties. + SchemaPropertyIterator schema3_iterator(schema_type_config3, type_config_map); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("prop1.prop1.prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("prop1.prop1.prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), + Eq("prop1.prop1.prop3")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), Eq("prop1.prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), Eq("prop1.prop3")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), Eq("prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config3.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), Eq("prop3.prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), Eq("prop3.prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), Eq("prop3.prop3")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), Eq("prop4.prop1")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), Eq("prop4.prop2")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), IsOk()); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyPath(), Eq("prop4.prop3")); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema3_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema3_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema3_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration for Schema2: + // {"prop1.prop1", "prop1.prop2", + // "prop1.prop3", "prop2", "prop3"} + // + // Indexable properties: + // {"prop1.prop2", "prop1.prop3", "prop2", "prop3"} + // + // Indexable_nested_properties set for Schema3.prop1 does not propagate + // to Schema2. + SchemaPropertyIterator schema2_iterator(schema_type_config2, type_config_map); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), Eq("prop1.prop1")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(0))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), Eq("prop1.prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), Eq("prop1.prop3")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config1.properties(2))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), Eq("prop2")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(1))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), IsOk()); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyPath(), Eq("prop3")); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config2.properties(2))); + EXPECT_THAT(schema2_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema2_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema2_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); +} TEST(SchemaPropertyIteratorTest, SingleLevelCycle) { std::string schema_a = "A"; std::string schema_b = "B"; @@ -457,17 +1929,20 @@ TEST(SchemaPropertyIteratorTest, SingleLevelCycle) { Eq("schemaAprop1.schemaBprop2")); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_b.properties(1))); - EXPECT_THAT(schema_a_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), Eq("schemaAprop2")); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_a.properties(1))); - EXPECT_THAT(schema_a_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_a_iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + EXPECT_THAT(schema_a_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + // Order of iteration for schema B: // {"schemaBprop2"}, indexable. SchemaPropertyIterator schema_b_iterator(schema_type_config_b, @@ -477,10 +1952,13 @@ TEST(SchemaPropertyIteratorTest, SingleLevelCycle) { EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), Eq("schemaBprop2")); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_b.properties(1))); - EXPECT_THAT(schema_b_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_b_iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_b_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); } TEST(SchemaPropertyIteratorTest, MultipleLevelCycle) { @@ -542,24 +2020,27 @@ TEST(SchemaPropertyIteratorTest, MultipleLevelCycle) { Eq("schemaAprop1.schemaBprop1.schemaCprop2")); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_c.properties(1))); - EXPECT_THAT(schema_a_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), Eq("schemaAprop1.schemaBprop2")); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_b.properties(1))); - EXPECT_THAT(schema_a_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), Eq("schemaAprop2")); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_a.properties(1))); - EXPECT_THAT(schema_a_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_a_iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + EXPECT_THAT(schema_a_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + // Order of iteration for schema B: // {"schemaBprop1.schemaCprop1.schemaAprop2", "schemaBprop1.schemaCprop2", // "schemaBprop2"} @@ -573,24 +2054,27 @@ TEST(SchemaPropertyIteratorTest, MultipleLevelCycle) { Eq("schemaBprop1.schemaCprop1.schemaAprop2")); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_a.properties(1))); - EXPECT_THAT(schema_b_iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), Eq("schemaBprop1.schemaCprop2")); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_c.properties(1))); - EXPECT_THAT(schema_b_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), Eq("schemaBprop2")); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_b.properties(1))); - EXPECT_THAT(schema_b_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_b_iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + EXPECT_THAT(schema_b_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + // Order of iteration for schema C: // {"schemaCprop1.schemaAprop1.schemaBprop2", "schemaCprop1.schemaAprop2", // "schemaCprop2"} @@ -604,23 +2088,222 @@ TEST(SchemaPropertyIteratorTest, MultipleLevelCycle) { Eq("schemaCprop1.schemaAprop1.schemaBprop2")); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_b.properties(1))); - EXPECT_THAT(schema_c_iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), Eq("schemaCprop1.schemaAprop2")); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_a.properties(1))); - EXPECT_THAT(schema_c_iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), Eq("schemaCprop2")); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_c.properties(1))); - EXPECT_THAT(schema_c_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_c_iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_c_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); +} + +TEST(SchemaPropertyIteratorTest, SingleLevelCycleWithIndexableList) { + std::string schema_a = "A"; + std::string schema_b = "B"; + + // Create schema with A -> B -> B -> B... + SchemaTypeConfigProto schema_type_config_a = + SchemaTypeConfigBuilder() + .SetType(schema_a) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaAprop1") + .SetDataTypeDocument( + schema_b, /*index_nested_properties=*/true)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaAprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config_b = + SchemaTypeConfigBuilder() + .SetType(schema_b) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaBprop1") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaBprop2") + .SetDataTypeDocument( + schema_b, /*indexable_nested_properties_list=*/ + {"schemaBprop1", "schemaBprop2.schemaBprop1", + "schemaBprop2.schemaBprop3", + "schemaBprop2.schemaBprop2.schemaBprop3"})) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaBprop3") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + + SchemaUtil::TypeConfigMap type_config_map = { + {schema_a, schema_type_config_a}, {schema_b, schema_type_config_b}}; + + // Order of iteration and whether each property is indexable for schema A: + // {"schemaAprop1.schemaBprop1" (true), + // "schemaAprop1.schemaBprop2.schemaBprop1" (true), + // "schemaAprop1.schemaBprop2.schemaBprop2.schemaBprop1" (true), + // "schemaAprop1.schemaBprop2.schemaBprop2.schemaBprop2.schemaBprop1" (false), + // "schemaAprop1.schemaBprop2.schemaBprop2.schemaBprop2.schemaBprop3" (true), + // "schemaAprop1.schemaBprop2.schemaBprop2.schemaBprop3" (true), + // "schemaAprop1.schemaBprop2.schemaBprop3" (false), + // "schemaAprop1.schemaBprop3" (true), + // "schemaAprop2" (true)} + SchemaPropertyIterator schema_a_iterator(schema_type_config_a, + type_config_map); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(0))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop2.schemaBprop1")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(0))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop2.schemaBprop2.schemaBprop1")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(0))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop2.schemaBprop2.schemaBprop2.schemaBprop1")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(0))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop2.schemaBprop2.schemaBprop2.schemaBprop3")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(2))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop2.schemaBprop2.schemaBprop3")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(2))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop2.schemaBprop3")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(2))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop3")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(2))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), Eq("schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_a_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration for schema B: + // {"schemaBprop1" (true), + // "schemaBprop2.schemaBprop1" (true), + // "schemaBprop2.schemaBprop2.schemaBprop1" (true), + // "schemaBprop2.schemaBprop2.schemaBprop2.schemaBprop1" (false), + // "schemaBprop2.schemaBprop2.schemaBprop2.schemaBprop3" (true), + // "schemaBprop2.schemaBprop2.schemaBprop3" (true), + // "schemaBprop2.schemaBprop3" (false), + // "schemaBprop3" (true)} + SchemaPropertyIterator schema_b_iterator(schema_type_config_b, + type_config_map); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), Eq("schemaBprop1")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(0))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop2.schemaBprop1")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(0))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop2.schemaBprop2.schemaBprop1")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(0))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop2.schemaBprop2.schemaBprop2.schemaBprop1")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(0))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop2.schemaBprop2.schemaBprop2.schemaBprop3")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(2))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop2.schemaBprop2.schemaBprop3")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(2))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop2.schemaBprop3")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(2))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), Eq("schemaBprop3")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(2))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_b_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); } TEST(SchemaPropertyIteratorTest, MultipleCycles) { @@ -629,7 +2312,11 @@ TEST(SchemaPropertyIteratorTest, MultipleCycles) { std::string schema_c = "C"; std::string schema_d = "D"; - // Create schema with D <-> A -> B -> C -> A -> B -> C -> A... + // Create the following schema: + // D <--> A <--- C + // \ ^ + // v / + // B // Schema type A has two cycles: A-B-C-A and A-D-A SchemaTypeConfigProto schema_type_config_a = SchemaTypeConfigBuilder() @@ -701,31 +2388,34 @@ TEST(SchemaPropertyIteratorTest, MultipleCycles) { Eq("schemaAprop1.schemaBprop1.schemaCprop2")); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_c.properties(1))); - EXPECT_THAT(schema_a_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), Eq("schemaAprop1.schemaBprop2")); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_b.properties(1))); - EXPECT_THAT(schema_a_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), Eq("schemaAprop2")); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_a.properties(1))); - EXPECT_THAT(schema_a_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), Eq("schemaAprop3.schemaDprop2")); EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_d.properties(1))); - EXPECT_THAT(schema_a_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_a_iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + EXPECT_THAT(schema_a_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + // Order of iteration for schema B: // {"schemaBprop1.schemaCprop1.schemaAprop2", // "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2", @@ -740,31 +2430,34 @@ TEST(SchemaPropertyIteratorTest, MultipleCycles) { Eq("schemaBprop1.schemaCprop1.schemaAprop2")); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_a.properties(1))); - EXPECT_THAT(schema_b_iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), Eq("schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2")); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_d.properties(1))); - EXPECT_THAT(schema_b_iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), Eq("schemaBprop1.schemaCprop2")); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_c.properties(1))); - EXPECT_THAT(schema_b_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), Eq("schemaBprop2")); EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_b.properties(1))); - EXPECT_THAT(schema_b_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_b_iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + EXPECT_THAT(schema_b_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + // Order of iteration for schema C: // {"schemaCprop1.schemaAprop1.schemaBprop2", "schemaCprop1.schemaAprop2", // "schemaCprop1.schemaAprop3.schemaDprop2", "schemaCprop2"} @@ -778,31 +2471,34 @@ TEST(SchemaPropertyIteratorTest, MultipleCycles) { Eq("schemaCprop1.schemaAprop1.schemaBprop2")); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_b.properties(1))); - EXPECT_THAT(schema_c_iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), Eq("schemaCprop1.schemaAprop2")); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_a.properties(1))); - EXPECT_THAT(schema_c_iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), Eq("schemaCprop1.schemaAprop3.schemaDprop2")); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_d.properties(1))); - EXPECT_THAT(schema_c_iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), Eq("schemaCprop2")); EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_c.properties(1))); - EXPECT_THAT(schema_c_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_c_iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + EXPECT_THAT(schema_c_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + // Order of iteration for schema D: // {"schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2", // "schemaDprop1.schemaAprop1.schemaBprop2", "schemaDprop1.schemaAprop2", @@ -817,30 +2513,1390 @@ TEST(SchemaPropertyIteratorTest, MultipleCycles) { Eq("schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2")); EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_c.properties(1))); - EXPECT_THAT(schema_d_iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), Eq("schemaDprop1.schemaAprop1.schemaBprop2")); EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_b.properties(1))); - EXPECT_THAT(schema_d_iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), Eq("schemaDprop1.schemaAprop2")); EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_a.properties(1))); - EXPECT_THAT(schema_d_iterator.GetCurrentNestedIndexable(), IsFalse()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsFalse()); EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), Eq("schemaDprop2")); EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), EqualsProto(schema_type_config_d.properties(1))); - EXPECT_THAT(schema_d_iterator.GetCurrentNestedIndexable(), IsTrue()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); EXPECT_THAT(schema_d_iterator.Advance(), StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_d_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); +} + +TEST(SchemaPropertyIteratorTest, MultipleCyclesWithIndexableList) { + std::string schema_a = "A"; + std::string schema_b = "B"; + std::string schema_c = "C"; + std::string schema_d = "D"; + + // Create the following schema: + // D <--> A <--- C + // \ ^ + // v / + // B + // Schema type A has two cycles: A-B-C-A and A-D-A + SchemaTypeConfigProto schema_type_config_a = + SchemaTypeConfigBuilder() + .SetType(schema_a) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaAprop1") + .SetDataTypeDocument( + schema_b, /*indexable_nested_properties_list=*/ + {"schemaBprop2", "schemaBprop1.schemaCprop1.schemaAprop2", + "schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2", + "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2", + "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1." + "schemaAprop2"})) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaAprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaAprop3") + .SetDataTypeDocument( + schema_d, /*indexable_nested_properties_list=*/ + {"schemaDprop2", "schemaDprop1.schemaAprop2", + "schemaDprop1.schemaAprop1.schemaBprop2", + "schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2", + "schemaDprop1.schemaAprop3.schemaDprop2"})) + .Build(); + SchemaTypeConfigProto schema_type_config_b = + SchemaTypeConfigBuilder() + .SetType(schema_b) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaBprop1") + .SetDataTypeDocument( + schema_c, /*index_nested_properties=*/true)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaBprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config_c = + SchemaTypeConfigBuilder() + .SetType(schema_c) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaCprop1") + .SetDataTypeDocument( + schema_a, /*index_nested_properties=*/false)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaCprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config_d = + SchemaTypeConfigBuilder() + .SetType(schema_d) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaDprop1") + .SetDataTypeDocument( + schema_a, /*index_nested_properties=*/false)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaDprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + + SchemaUtil::TypeConfigMap type_config_map = { + {schema_a, schema_type_config_a}, + {schema_b, schema_type_config_b}, + {schema_c, schema_type_config_c}, + {schema_d, schema_type_config_d}}; + + // Order of iteration and whether each property is indexable for schema A: + // "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2" (true), + // "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop2" (true), + // "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop2" + // (true), "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2" + // (true), "schemaAprop1.schemaBprop1.schemaCprop2" (false), + // "schemaAprop1.schemaBprop2" (true), + // "schemaAprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop3.schemaDprop2" (true), + // "schemaAprop3.schemaDprop2" (true) + SchemaPropertyIterator schema_a_iterator(schema_type_config_a, + type_config_map); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3." + "schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), Eq("schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_a_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration and whether each property is indexable for schema B: + // "schemaBprop1.schemaCprop1.schemaAprop2" (false), + // "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2" (false), + // "schemaBprop1.schemaCprop2" (true), + // "schemaBprop2" (true) + SchemaPropertyIterator schema_b_iterator(schema_type_config_b, + type_config_map); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), Eq("schemaBprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_b_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration for schema C: + // "schemaCprop1.schemaAprop1.schemaBprop2" (false), + // "schemaCprop1.schemaAprop2" (false), + // "schemaCprop1.schemaAprop3.schemaDprop2" (false), + // "schemaCprop2" (true) + SchemaPropertyIterator schema_c_iterator(schema_type_config_c, + type_config_map); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), Eq("schemaCprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_c_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration for schema D: + // "schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2" (false), + // "schemaDprop1.schemaAprop1.schemaBprop2" (false), + // "schemaDprop1.schemaAprop2" (false), + // "schemaDprop2" (true) + SchemaPropertyIterator schema_d_iterator(schema_type_config_d, + type_config_map); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), Eq("schemaDprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_d_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); +} + +TEST(SchemaPropertyIteratorTest, MultipleCyclesWithIndexableList_allIndexTrue) { + std::string schema_a = "A"; + std::string schema_b = "B"; + std::string schema_c = "C"; + std::string schema_d = "D"; + + // Create the following schema: + // D <--> A <--- C + // \ ^ + // v / + // B + // Schema type A has two cycles: A-B-C-A and A-D-A + SchemaTypeConfigProto schema_type_config_a = + SchemaTypeConfigBuilder() + .SetType(schema_a) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaAprop1") + .SetDataTypeDocument( + schema_b, /*indexable_nested_properties_list=*/ + {"schemaBprop2", "schemaBprop1.schemaCprop1.schemaAprop2", + "schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2", + "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2", + "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1." + "schemaAprop2"})) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaAprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaAprop3") + .SetDataTypeDocument( + schema_d, /*indexable_nested_properties_list=*/ + {"schemaDprop2", "schemaDprop1.schemaAprop2", + "schemaDprop1.schemaAprop1.schemaBprop2", + "schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2", + "schemaDprop1.schemaAprop3.schemaDprop2"})) + .Build(); + SchemaTypeConfigProto schema_type_config_b = + SchemaTypeConfigBuilder() + .SetType(schema_b) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaBprop1") + .SetDataTypeDocument( + schema_c, /*index_nested_properties=*/true)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaBprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config_c = + SchemaTypeConfigBuilder() + .SetType(schema_c) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaCprop1") + .SetDataTypeDocument( + schema_a, /*index_nested_properties=*/true)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaCprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config_d = + SchemaTypeConfigBuilder() + .SetType(schema_d) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaDprop1") + .SetDataTypeDocument( + schema_a, /*index_nested_properties=*/true)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaDprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + + SchemaUtil::TypeConfigMap type_config_map = { + {schema_a, schema_type_config_a}, + {schema_b, schema_type_config_b}, + {schema_c, schema_type_config_c}, + {schema_d, schema_type_config_d}}; + + // Order of iteration and whether each property is indexable for schema A: + // "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2" (true), + // "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop2" (true), + // "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop2" + // (true), "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2" + // (true), "schemaAprop1.schemaBprop1.schemaCprop2" (false), + // "schemaAprop1.schemaBprop2" (true), + // "schemaAprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop3.schemaDprop2" (true), + // "schemaAprop3.schemaDprop2" (true) + SchemaPropertyIterator schema_a_iterator(schema_type_config_a, + type_config_map); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3." + "schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), Eq("schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_a_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration and whether each property is indexable for schema B: + // "schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2" + // (true), + // "schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop2" + // (true), + // "schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop2" + // (true), + // "schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2" + // (true), "schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop2" + // (false), "schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2" (true), + // "schemaBprop1.schemaCprop1.schemaAprop2" (true), + // "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2" + // (true), + // "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop2" + // (true), "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop2" + // (true), + // "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop3.schemaDprop2" + // (true), "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2" (true) + // "schemaBprop1.schemaCprop2" (true) + // "schemaBprop2" (true) + + SchemaPropertyIterator schema_b_iterator(schema_type_config_b, + type_config_map); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop1." + "schemaCprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop1." + "schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop1." + "schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop1." + "schemaCprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1." + "schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1." + "schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1." + "schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), Eq("schemaBprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_b_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration and whether each property is indexable for schema C: + // "schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2" + // (true), "schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop2" + // (true), + // "schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop2" + // (true), + // "schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2" + // (true), + // "schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop2" (false), + // "schemaCprop1.schemaAprop1.schemaBprop2" (true), + // "schemaCprop1.schemaAprop2" (true), + // "schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2" + // (true), + // "schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop2" (true), + // "schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop2" (true), + // "schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop3.schemaDprop2" (true), + // "schemaCprop1.schemaAprop3.schemaDprop2" (true) + // "schemaCprop2" (true) + SchemaPropertyIterator schema_c_iterator(schema_type_config_c, + type_config_map); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1." + "schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1." + "schemaAprop3.schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop1." + "schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop1." + "schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), Eq("schemaCprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_c_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration and whether each property is indexable for schema D: + // "schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2" + // (true), "schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop2" + // (true), + // "schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop2" + // (true), + // "schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2" + // (true), "schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2" (false), + // "schemaDprop1.schemaAprop1.schemaBprop2" (true), + // "schemaDprop1.schemaAprop2" (true), + // "schemaDprop1.schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2" + // (true), "schemaDprop1.schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop2" + // (true), "schemaDprop1.schemaAprop3.schemaDprop1.schemaAprop2" (true), + // "schemaDprop1.schemaAprop3.schemaDprop1.schemaAprop3.schemaDprop2" (true), + // "schemaDprop1.schemaAprop3.schemaDprop2" (true), + // "schemaDprop2" (true) + SchemaPropertyIterator schema_d_iterator(schema_type_config_d, + type_config_map); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop1." + "schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop1." + "schemaAprop3.schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop1." + "schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop3.schemaDprop1.schemaAprop1." + "schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop3.schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop3.schemaDprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), Eq("schemaDprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_d_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); +} + +TEST(SchemaPropertyIteratorTest, + MultipleCyclesWithIndexableList_unknownPropPaths) { + std::string schema_a = "A"; + std::string schema_b = "B"; + std::string schema_c = "C"; + std::string schema_d = "D"; + + // Create the following schema: + // D <--> A <--- C + // \ ^ + // v / + // B + // Schema type A has two cycles: A-B-C-A and A-D-A + SchemaTypeConfigProto schema_type_config_a = + SchemaTypeConfigBuilder() + .SetType(schema_a) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaAprop1") + .SetDataTypeDocument( + schema_b, /*indexable_nested_properties_list=*/ + {"schemaBprop2", "schemaBprop1.schemaCprop1.schemaAprop2", + "schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2", + "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2", + "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1." + "schemaAprop2", + "schemaBprop1.schemaCprop1", + "schemaBprop1.schemaCprop1.schemaAprop3", "schemaAprop2", + "schemaBprop2.schemaCprop2", "schemaBprop1.foo.bar", + "foo", "foo", "bar"})) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaAprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaAprop3") + .SetDataTypeDocument( + schema_d, /*indexable_nested_properties_list=*/ + {"schemaDprop2", "schemaDprop1.schemaAprop2", + "schemaDprop1.schemaAprop1.schemaBprop2", + "schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2", + "schemaDprop1.schemaAprop3.schemaDprop2", "schemaBprop2", + "bar", "schemaDprop2.foo", "schemaDprop1", + "schemaAprop3.schemaDprop2"})) + .Build(); + SchemaTypeConfigProto schema_type_config_b = + SchemaTypeConfigBuilder() + .SetType(schema_b) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaBprop1") + .SetDataTypeDocument( + schema_c, /*index_nested_properties=*/true)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaBprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config_c = + SchemaTypeConfigBuilder() + .SetType(schema_c) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaCprop1") + .SetDataTypeDocument( + schema_a, /*index_nested_properties=*/false)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaCprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config_d = + SchemaTypeConfigBuilder() + .SetType(schema_d) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaDprop1") + .SetDataTypeDocument( + schema_a, /*index_nested_properties=*/false)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaDprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + + SchemaUtil::TypeConfigMap type_config_map = { + {schema_a, schema_type_config_a}, + {schema_b, schema_type_config_b}, + {schema_c, schema_type_config_c}, + {schema_d, schema_type_config_d}}; + + // Order of iteration and whether each property is indexable for schema A: + // "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2" (true), + // "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop2" (true), + // "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop1.schemaAprop2" + // (true), "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2" + // (true), "schemaAprop1.schemaBprop1.schemaCprop2" (false), + // "schemaAprop1.schemaBprop2" (true), + // "schemaAprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop2" (true), + // "schemaAprop3.schemaDprop1.schemaAprop3.schemaDprop2" (true), + // "schemaAprop3.schemaDprop2" (true) + // + // The following properties listed in the indexable_list are not defined + // in the schema and should not be seen during iteration. These should appear + // in the unknown_indexable_nested_properties_ set. + // "schemaAprop1.bar", + // "schemaAprop1.foo", + // "schemaAprop1.schemaAprop2", + // "schemaAprop1.schemaBprop1.foo.bar", + // "schemaAprop1.schemaBprop1.schemaCprop1", + // "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3", + // "schemaAprop1.schemaBprop2.schemaCprop2", + // "schemaAprop3.bar", + // "schemaAprop3.schemaAprop3.schemaDprop2", + // "schemaAprop3.schemaBprop2", + // "schemaAprop3.schemaDprop1", + // "schemaAprop3.schemaDprop2.foo" + SchemaPropertyIterator schema_a_iterator(schema_type_config_a, + type_config_map); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3." + "schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), Eq("schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT( + schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT( + schema_a_iterator.unknown_indexable_nested_property_paths(), + ElementsAre( + "schemaAprop1.bar", "schemaAprop1.foo", "schemaAprop1.schemaAprop2", + "schemaAprop1.schemaBprop1.foo.bar", + "schemaAprop1.schemaBprop1.schemaCprop1", + "schemaAprop1.schemaBprop1.schemaCprop1.schemaAprop3", + "schemaAprop1.schemaBprop2.schemaCprop2", "schemaAprop3.bar", + "schemaAprop3.schemaAprop3.schemaDprop2", "schemaAprop3.schemaBprop2", + "schemaAprop3.schemaDprop1", "schemaAprop3.schemaDprop2.foo")); + + // Order of iteration and whether each property is indexable for schema B: + // "schemaBprop1.schemaCprop1.schemaAprop2" (false), + // "schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2" (false), + // "schemaBprop1.schemaCprop2" (true), + // "schemaBprop2" (true) + SchemaPropertyIterator schema_b_iterator(schema_type_config_b, + type_config_map); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), + Eq("schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyPath(), Eq("schemaBprop2")); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_b_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_b_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_b_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration for schema C: + // "schemaCprop1.schemaAprop1.schemaBprop2" (false), + // "schemaCprop1.schemaAprop2" (false), + // "schemaCprop1.schemaAprop3.schemaDprop2" (false), + // "schemaCprop2" (true) + SchemaPropertyIterator schema_c_iterator(schema_type_config_c, + type_config_map); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), + Eq("schemaCprop1.schemaAprop3.schemaDprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_c_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyPath(), Eq("schemaCprop2")); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_c_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_c_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_c_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); + + // Order of iteration for schema D: + // "schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2" (false), + // "schemaDprop1.schemaAprop1.schemaBprop2" (false), + // "schemaDprop1.schemaAprop2" (false), + // "schemaDprop2" (true) + SchemaPropertyIterator schema_d_iterator(schema_type_config_d, + type_config_map); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop1.schemaBprop1.schemaCprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_c.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), + Eq("schemaDprop1.schemaAprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_d_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyPath(), Eq("schemaDprop2")); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_d.properties(1))); + EXPECT_THAT(schema_d_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_d_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_d_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); +} + +TEST(SchemaPropertyIteratorTest, TopLevelCycleWithMultipleIndexableLists) { + std::string schema_a = "A"; + std::string schema_b = "B"; + std::string schema_c = "C"; + std::string schema_d = "D"; + + // Create the following schema: + // A <-> A -> B + // A has a top-level property that is a self-reference. + SchemaTypeConfigProto schema_type_config_a = + SchemaTypeConfigBuilder() + .SetType(schema_a) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaAprop1") + .SetDataTypeDocument( + schema_b, /*indexable_nested_properties_list=*/ + {"schemaBprop1", "schemaBprop2"})) + .AddProperty(PropertyConfigBuilder() + .SetName("schemaAprop2") + .SetDataTypeDocument( + schema_a, /*indexable_nested_properties_list=*/ + {"schemaAprop1.schemaBprop2", + "schemaAprop1.schemaBprop3"})) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaAprop3") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + SchemaTypeConfigProto schema_type_config_b = + SchemaTypeConfigBuilder() + .SetType(schema_b) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaBprop1") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaBprop2") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .AddProperty( + PropertyConfigBuilder() + .SetName("schemaBprop3") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN)) + .Build(); + + SchemaUtil::TypeConfigMap type_config_map = { + {schema_a, schema_type_config_a}, {schema_b, schema_type_config_b}}; + + // Order of iteration for Schema A: + // "schemaAprop1.schemaBprop1" (true) + // "schemaAprop1.schemaBprop2" (true) + // "schemaAprop1.schemaBprop3" (false) + // "schemaAprop2.schemaAprop1.schemaBprop1" (false) + // "schemaAprop2.schemaAprop1.schemaBprop2" (true) + // "schemaAprop2.schemaAprop1.schemaBprop3" (true) + // "schemaAprop2.schemaAprop3" (false) + // "schemaAprop3" (true) + SchemaPropertyIterator schema_a_iterator(schema_type_config_a, + type_config_map); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop1")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(0))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop1.schemaBprop3")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(2))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop2.schemaAprop1.schemaBprop1")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(0))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop2.schemaAprop1.schemaBprop2")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(1))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop2.schemaAprop1.schemaBprop3")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_b.properties(2))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), + Eq("schemaAprop2.schemaAprop3")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(2))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsFalse()); + + EXPECT_THAT(schema_a_iterator.Advance(), IsOk()); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyPath(), Eq("schemaAprop3")); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyConfig(), + EqualsProto(schema_type_config_a.properties(2))); + EXPECT_THAT(schema_a_iterator.GetCurrentPropertyIndexable(), IsTrue()); + + EXPECT_THAT(schema_a_iterator.Advance(), + StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); + + EXPECT_THAT(schema_a_iterator.unknown_indexable_nested_property_paths(), + IsEmpty()); } } // namespace diff --git a/icing/schema/schema-store.cc b/icing/schema/schema-store.cc index bcc7c2c..85ee6b6 100644 --- a/icing/schema/schema-store.cc +++ b/icing/schema/schema-store.cc @@ -448,7 +448,7 @@ libtextclassifier3::Status SchemaStore::InitializeDerivedFiles() { "Combined checksum of SchemaStore was inconsistent"); } - BuildInMemoryCache(); + ICING_RETURN_IF_ERROR(BuildInMemoryCache()); return libtextclassifier3::Status::OK; } @@ -463,7 +463,7 @@ libtextclassifier3::Status SchemaStore::RegenerateDerivedFiles( ICING_RETURN_IF_ERROR(schema_type_mapper_->Put( type_config.schema_type(), schema_type_mapper_->num_keys())); } - BuildInMemoryCache(); + ICING_RETURN_IF_ERROR(BuildInMemoryCache()); if (create_overlay_if_necessary) { ICING_ASSIGN_OR_RETURN( @@ -485,12 +485,16 @@ libtextclassifier3::Status SchemaStore::RegenerateDerivedFiles( std::make_unique<SchemaProto>(std::move(base_schema)); ICING_RETURN_IF_ERROR(schema_file_->Write(std::move(base_schema_ptr))); + // LINT.IfChange(min_overlay_version_compatibility) + // Although the current version is 2, the schema is compatible with + // version 1, so min_overlay_version_compatibility should be 1. + int32_t min_overlay_version_compatibility = version_util::kVersionOne; + // LINT.ThenChange(//depot/google3/icing/file/version-util.h:kVersion) header_->SetOverlayInfo( - /*overlay_created=*/true, - /*min_overlay_version_compatibility=*/version_util::kVersionOne); + /*overlay_created=*/true, min_overlay_version_compatibility); // Rebuild in memory data - references to the old schema will be invalid // now. - BuildInMemoryCache(); + ICING_RETURN_IF_ERROR(BuildInMemoryCache()); } } @@ -776,6 +780,17 @@ libtextclassifier3::StatusOr<SchemaTypeId> SchemaStore::GetSchemaTypeId( return schema_type_mapper_->Get(schema_type); } +libtextclassifier3::StatusOr<const std::string*> SchemaStore::GetSchemaType( + SchemaTypeId schema_type_id) const { + ICING_RETURN_IF_ERROR(CheckSchemaSet()); + if (const auto it = reverse_schema_type_mapper_.find(schema_type_id); + it == reverse_schema_type_mapper_.end()) { + return absl_ports::InvalidArgumentError("Invalid schema type id"); + } else { + return &it->second; + } +} + libtextclassifier3::StatusOr<const std::unordered_set<SchemaTypeId>*> SchemaStore::GetSchemaTypeIdsWithChildren(std::string_view schema_type) const { ICING_ASSIGN_OR_RETURN(SchemaTypeId schema_type_id, diff --git a/icing/schema/schema-store.h b/icing/schema/schema-store.h index 6075f5b..88968b1 100644 --- a/icing/schema/schema-store.h +++ b/icing/schema/schema-store.h @@ -276,6 +276,15 @@ class SchemaStore { libtextclassifier3::StatusOr<const SchemaTypeConfigProto*> GetSchemaTypeConfig(std::string_view schema_type) const; + // Returns the schema type of the passed in SchemaTypeId + // + // Returns: + // schema type on success + // FAILED_PRECONDITION if schema hasn't been set yet + // INVALID_ARGUMENT if schema type id is invalid + libtextclassifier3::StatusOr<const std::string*> GetSchemaType( + SchemaTypeId schema_type_id) const; + // Returns the SchemaTypeId of the passed in schema type // // Returns: diff --git a/icing/schema/schema-store_test.cc b/icing/schema/schema-store_test.cc index 3298b75..8cc7008 100644 --- a/icing/schema/schema-store_test.cc +++ b/icing/schema/schema-store_test.cc @@ -1084,6 +1084,137 @@ TEST_F(SchemaStoreTest, SetSchemaWithCompatibleNestedTypesOk) { EXPECT_THAT(*actual_schema, EqualsProto(new_schema)); } +TEST_F(SchemaStoreTest, SetSchemaWithAddedIndexableNestedTypeOk) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + + // 1. Create a ContactPoint type with a optional property, and a type that + // references the ContactPoint type. + SchemaTypeConfigBuilder contact_point = + SchemaTypeConfigBuilder() + .SetType("ContactPoint") + .AddProperty( + PropertyConfigBuilder() + .SetName("label") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_REPEATED)); + SchemaTypeConfigBuilder person = + SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("contactPoints") + .SetDataTypeDocument("ContactPoint", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_REPEATED)); + SchemaProto old_schema = + SchemaBuilder().AddType(contact_point).AddType(person).Build(); + ICING_EXPECT_OK(schema_store->SetSchema( + old_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + // 2. Add another nested document property to "Person" that has type + // "ContactPoint" + SchemaTypeConfigBuilder new_person = + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("contactPoints") + .SetDataTypeDocument("ContactPoint", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_REPEATED)) + .AddProperty( + PropertyConfigBuilder() + .SetName("anotherContactPoint") + .SetDataTypeDocument("ContactPoint", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_REPEATED)); + SchemaProto new_schema = + SchemaBuilder().AddType(contact_point).AddType(new_person).Build(); + + // 3. Set to new schema. "Person" should be index-incompatible since we need + // to index an additional property: 'anotherContactPoint.label'. + // - "Person" is also considered join-incompatible since the added nested + // document property could also contain a joinable property. + SchemaStore::SetSchemaResult expected_result; + expected_result.success = true; + expected_result.schema_types_index_incompatible_by_name.insert("Person"); + expected_result.schema_types_join_incompatible_by_name.insert("Person"); + + EXPECT_THAT(schema_store->SetSchema( + new_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(expected_result))); + ICING_ASSERT_OK_AND_ASSIGN(const SchemaProto* actual_schema, + schema_store->GetSchema()); + EXPECT_THAT(*actual_schema, EqualsProto(new_schema)); +} + +TEST_F(SchemaStoreTest, SetSchemaWithAddedJoinableNestedTypeOk) { + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaStore> schema_store, + SchemaStore::Create(&filesystem_, schema_store_dir_, &fake_clock_)); + + // 1. Create a ContactPoint type with a optional property, and a type that + // references the ContactPoint type. + SchemaTypeConfigBuilder contact_point = + SchemaTypeConfigBuilder() + .SetType("ContactPoint") + .AddProperty( + PropertyConfigBuilder() + .SetName("label") + .SetDataTypeString(TERM_MATCH_PREFIX, TOKENIZER_PLAIN) + .SetJoinable(JOINABLE_VALUE_TYPE_QUALIFIED_ID, + /*propagate_delete=*/false) + .SetCardinality(CARDINALITY_REQUIRED)); + SchemaTypeConfigBuilder person = + SchemaTypeConfigBuilder().SetType("Person").AddProperty( + PropertyConfigBuilder() + .SetName("contactPoints") + .SetDataTypeDocument("ContactPoint", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)); + SchemaProto old_schema = + SchemaBuilder().AddType(contact_point).AddType(person).Build(); + ICING_EXPECT_OK(schema_store->SetSchema( + old_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false)); + + // 2. Add another nested document property to "Person" that has type + // "ContactPoint", but make it non-indexable + SchemaTypeConfigBuilder new_person = + SchemaTypeConfigBuilder() + .SetType("Person") + .AddProperty( + PropertyConfigBuilder() + .SetName("contactPoints") + .SetDataTypeDocument("ContactPoint", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("anotherContactPoint") + .SetDataTypeDocument("ContactPoint", + /*index_nested_properties=*/false) + .SetCardinality(CARDINALITY_OPTIONAL)); + SchemaProto new_schema = + SchemaBuilder().AddType(contact_point).AddType(new_person).Build(); + + // 3. Set to new schema. "Person" should be join-incompatible but + // index-compatible. + SchemaStore::SetSchemaResult expected_result; + expected_result.success = true; + expected_result.schema_types_join_incompatible_by_name.insert("Person"); + + EXPECT_THAT(schema_store->SetSchema( + new_schema, /*ignore_errors_and_delete_documents=*/false, + /*allow_circular_schema_definitions=*/false), + IsOkAndHolds(EqualsSetSchemaResult(expected_result))); + ICING_ASSERT_OK_AND_ASSIGN(const SchemaProto* actual_schema, + schema_store->GetSchema()); + EXPECT_THAT(*actual_schema, EqualsProto(new_schema)); +} + TEST_F(SchemaStoreTest, GetSchemaTypeId) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaStore> schema_store, @@ -2722,7 +2853,8 @@ TEST_F(SchemaStoreTest, MigrateSchemaVersionZeroUpgradeNoChange) { } } -TEST_F(SchemaStoreTest, MigrateSchemaRollbackDiscardsOverlaySchema) { +TEST_F(SchemaStoreTest, + MigrateSchemaRollbackDiscardsIncompatibleOverlaySchema) { // Because we are upgrading from version zero, the schema must be compatible // with version zero. SchemaTypeConfigProto type_a = @@ -2749,12 +2881,12 @@ TEST_F(SchemaStoreTest, MigrateSchemaRollbackDiscardsOverlaySchema) { IsOkAndHolds(Pointee(EqualsProto(schema)))); } - // Rollback to a version before kVersion. The schema header will declare that - // the overlay is compatible with any version starting with kVersion. So - // kVersion - 1 is incompatible and will throw out the schema. + // Rollback to a version before kVersionOne. The schema header will declare + // that the overlay is compatible with any version starting with kVersionOne. + // So kVersionOne - 1 is incompatible and will throw out the schema. ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kRollBack, - version_util::kVersion - 1)); + version_util::kVersionOne - 1)); { // Create a new of the schema store and check that we fell back to the @@ -2777,7 +2909,7 @@ TEST_F(SchemaStoreTest, MigrateSchemaRollbackDiscardsOverlaySchema) { } } -TEST_F(SchemaStoreTest, MigrateSchemaCompatibleRollbackKeepsOverlaySchema) { +TEST_F(SchemaStoreTest, MigrateSchemaRollbackKeepsCompatibleOverlaySchema) { // Because we are upgrading from version zero, the schema must be compatible // with version zero. SchemaTypeConfigProto type_a = @@ -2846,12 +2978,12 @@ TEST_F(SchemaStoreTest, MigrateSchemaRollforwardRetainsBaseSchema) { IsOkAndHolds(Pointee(EqualsProto(schema)))); } - // Rollback to a version before kVersion. The schema header will declare that - // the overlay is compatible with any version starting with kVersion. So - // kVersion - 1 is incompatible and will throw out the schema. + // Rollback to a version before kVersionOne. The schema header will declare + // that the overlay is compatible with any version starting with kVersionOne. + // So kVersionOne - 1 is incompatible and will throw out the schema. ICING_EXPECT_OK(SchemaStore::MigrateSchema( &filesystem_, schema_store_dir_, version_util::StateChange::kRollBack, - version_util::kVersion - 1)); + version_util::kVersionOne - 1)); SchemaTypeConfigProto other_type_a = SchemaTypeConfigBuilder() diff --git a/icing/schema/schema-type-manager.cc b/icing/schema/schema-type-manager.cc index f3a86d4..4a6b7f2 100644 --- a/icing/schema/schema-type-manager.cc +++ b/icing/schema/schema-type-manager.cc @@ -20,6 +20,7 @@ #include "icing/text_classifier/lib3/utils/base/statusor.h" #include "icing/absl_ports/canonical_errors.h" #include "icing/schema/joinable-property-manager.h" +#include "icing/schema/property-util.h" #include "icing/schema/schema-property-iterator.h" #include "icing/schema/schema-util.h" #include "icing/schema/section-manager.h" @@ -55,7 +56,7 @@ SchemaTypeManager::Create(const SchemaUtil::TypeConfigMap& type_config_map, } // Process section (indexable property) - if (iterator.GetCurrentNestedIndexable()) { + if (iterator.GetCurrentPropertyIndexable()) { ICING_RETURN_IF_ERROR( section_manager_builder.ProcessSchemaTypePropertyConfig( schema_type_id, iterator.GetCurrentPropertyConfig(), @@ -68,6 +69,34 @@ SchemaTypeManager::Create(const SchemaUtil::TypeConfigMap& type_config_map, schema_type_id, iterator.GetCurrentPropertyConfig(), iterator.GetCurrentPropertyPath())); } + + // Process unknown property paths in the indexable_nested_properties_list. + // These property paths should consume sectionIds but are currently + // not indexed. + // + // SectionId assignment order: + // - We assign section ids to known (existing) properties first in alphabet + // order. + // - After handling all known properties, we assign section ids to all + // unknown (non-existent) properties that are specified in the + // indexable_nested_properties_list. + // - As a result, assignment of the entire section set is not done + // alphabetically, but assignment is still deterministic and alphabetical + // order is preserved inside the known properties and unknown properties + // sets individually. + for (const auto& property_path : + iterator.unknown_indexable_nested_property_paths()) { + PropertyConfigProto unknown_property_config; + unknown_property_config.set_property_name(std::string( + property_util::SplitPropertyPathExpr(property_path).back())); + unknown_property_config.set_data_type( + PropertyConfigProto::DataType::UNKNOWN); + + ICING_RETURN_IF_ERROR( + section_manager_builder.ProcessSchemaTypePropertyConfig( + schema_type_id, unknown_property_config, + std::string(property_path))); + } } return std::unique_ptr<SchemaTypeManager>(new SchemaTypeManager( diff --git a/icing/schema/schema-util.cc b/icing/schema/schema-util.cc index 371ed00..af6feda 100644 --- a/icing/schema/schema-util.cc +++ b/icing/schema/schema-util.cc @@ -115,6 +115,34 @@ bool IsIntegerNumericMatchTypeCompatible( return old_indexed.numeric_match_type() == new_indexed.numeric_match_type(); } +bool IsDocumentIndexingCompatible(const DocumentIndexingConfig& old_indexed, + const DocumentIndexingConfig& new_indexed) { + // TODO(b/265304217): This could mark the new schema as incompatible and + // generate some unnecessary index rebuilds if the two schemas have an + // equivalent set of indexed properties, but changed the way that it is + // declared. + if (old_indexed.index_nested_properties() != + new_indexed.index_nested_properties()) { + return false; + } + + if (old_indexed.indexable_nested_properties_list().size() != + new_indexed.indexable_nested_properties_list().size()) { + return false; + } + + std::unordered_set<std::string_view> old_indexable_nested_properies_set( + old_indexed.indexable_nested_properties_list().begin(), + old_indexed.indexable_nested_properties_list().end()); + for (const auto& property : new_indexed.indexable_nested_properties_list()) { + if (old_indexable_nested_properies_set.find(property) == + old_indexable_nested_properies_set.end()) { + return false; + } + } + return true; +} + void AddIncompatibleChangeToDelta( std::unordered_set<std::string>& incompatible_delta, const SchemaTypeConfigProto& old_type_config, @@ -252,8 +280,6 @@ libtextclassifier3::Status CalculateTransitiveNestedTypeRelations( // 4. "adjacent" has been fully expanded. Add all of its transitive // outgoing relations to this type's transitive outgoing relations. auto adjacent_expanded_itr = expanded_nested_types_map->find(adjacent_type); - expanded_relations.reserve(expanded_relations.size() + - adjacent_expanded_itr->second.size()); for (const auto& [transitive_reachable, _] : adjacent_expanded_itr->second) { // Insert a transitive reachable node `transitive_reachable` for `type` if @@ -317,8 +343,6 @@ libtextclassifier3::Status CalculateAcyclicTransitiveRelations( // 3. "adjacent" has been fully expanded. Add all of its transitive outgoing // relations to this type's transitive outgoing relations. auto adjacent_expanded_itr = expanded_relation_map->find(adjacent); - expanded_relations.reserve(expanded_relations.size() + - adjacent_expanded_itr->second.size()); for (const auto& [transitive_reachable, _] : adjacent_expanded_itr->second) { // Insert a transitive reachable node `transitive_reachable` for `type`. @@ -498,7 +522,6 @@ BuildTransitiveDependentGraph(const SchemaProto& schema, // Insert the parent_type into the dependent map if it is not present // already. merged_dependent_map.insert({parent_type, {}}); - merged_dependent_map[parent_type].reserve(inheritance_relation.size()); for (const auto& [child_type, _] : inheritance_relation) { // Insert the child_type into parent_type's dependent map if it's not // present already, in which case the value will be an empty vector. @@ -571,6 +594,10 @@ libtextclassifier3::StatusOr<SchemaUtil::DependentMap> SchemaUtil::Validate( "data_types in schema property '", schema_type, ".", property_name, "'")); } + + ICING_RETURN_IF_ERROR(ValidateDocumentIndexingConfig( + property_config.document_indexing_config(), schema_type, + property_name)); } ICING_RETURN_IF_ERROR(ValidateCardinality(property_config.cardinality(), @@ -751,6 +778,20 @@ libtextclassifier3::Status SchemaUtil::ValidateJoinableConfig( return libtextclassifier3::Status::OK; } +libtextclassifier3::Status SchemaUtil::ValidateDocumentIndexingConfig( + const DocumentIndexingConfig& config, std::string_view schema_type, + std::string_view property_name) { + if (!config.indexable_nested_properties_list().empty() && + config.index_nested_properties()) { + return absl_ports::InvalidArgumentError(absl_ports::StrCat( + "DocumentIndexingConfig.index_nested_properties is required to be " + "false when providing a non-empty indexable_nested_properties_list " + "for property '", + schema_type, ".", property_name, "'")); + } + return libtextclassifier3::Status::OK; +} + /* static */ bool SchemaUtil::IsIndexedProperty( const PropertyConfigProto& property_config) { switch (property_config.data_type()) { @@ -762,11 +803,19 @@ libtextclassifier3::Status SchemaUtil::ValidateJoinableConfig( case PropertyConfigProto::DataType::INT64: return property_config.integer_indexing_config().numeric_match_type() != IntegerIndexingConfig::NumericMatchType::UNKNOWN; + case PropertyConfigProto::DataType::DOCUMENT: + // A document property is considered indexed if it has + // index_nested_properties=true, or a non-empty + // indexable_nested_properties_list. + return property_config.document_indexing_config() + .index_nested_properties() || + !property_config.document_indexing_config() + .indexable_nested_properties_list() + .empty(); case PropertyConfigProto::DataType::UNKNOWN: case PropertyConfigProto::DataType::DOUBLE: case PropertyConfigProto::DataType::BOOLEAN: case PropertyConfigProto::DataType::BYTES: - case PropertyConfigProto::DataType::DOCUMENT: return false; } } @@ -899,6 +948,13 @@ SchemaUtil::ParsedPropertyConfigs SchemaUtil::ParsePropertyConfigs( JoinableConfig::ValueType::NONE) { ++parsed_property_configs.num_joinable_properties; } + + // Also keep track of how many nested document properties there are. Adding + // new nested document properties will result in join-index rebuild. + if (property_config.data_type() == + PropertyConfigProto::DataType::DOCUMENT) { + ++parsed_property_configs.num_nested_document_properties; + } } return parsed_property_configs; @@ -937,6 +993,7 @@ const SchemaUtil::SchemaDelta SchemaUtil::ComputeCompatibilityDelta( int32_t old_required_properties = 0; int32_t old_indexed_properties = 0; int32_t old_joinable_properties = 0; + int32_t old_nested_document_properties = 0; // If there is a different number of properties, then there must have been a // change. @@ -966,6 +1023,14 @@ const SchemaUtil::SchemaDelta SchemaUtil::ComputeCompatibilityDelta( ++old_joinable_properties; } + // A nested-document property is a property of DataType::DOCUMENT. + bool is_nested_document_property = + old_property_config.data_type() == + PropertyConfigProto::DataType::DOCUMENT; + if (is_nested_document_property) { + ++old_nested_document_properties; + } + auto new_property_name_and_config = new_parsed_property_configs.property_config_map.find( old_property_config.property_name()); @@ -979,7 +1044,8 @@ const SchemaUtil::SchemaDelta SchemaUtil::ComputeCompatibilityDelta( "' was not defined in new schema"); is_incompatible = true; is_index_incompatible |= is_indexed_property; - is_join_incompatible |= is_joinable_property; + is_join_incompatible |= + is_joinable_property || is_nested_document_property; continue; } @@ -1005,10 +1071,9 @@ const SchemaUtil::SchemaDelta SchemaUtil::ComputeCompatibilityDelta( !IsIntegerNumericMatchTypeCompatible( old_property_config.integer_indexing_config(), new_property_config->integer_indexing_config()) || - old_property_config.document_indexing_config() - .index_nested_properties() != - new_property_config->document_indexing_config() - .index_nested_properties()) { + !IsDocumentIndexingCompatible( + old_property_config.document_indexing_config(), + new_property_config->document_indexing_config())) { is_index_incompatible = true; } @@ -1032,8 +1097,9 @@ const SchemaUtil::SchemaDelta SchemaUtil::ComputeCompatibilityDelta( is_incompatible = true; } - // If we've gained any new indexed properties, then the section ids may - // change. Since the section ids are stored in the index, we'll need to + // If we've gained any new indexed properties (this includes gaining new + // indexed nested document properties), then the section ids may change. + // Since the section ids are stored in the index, we'll need to // reindex everything. if (new_parsed_property_configs.num_indexed_properties > old_indexed_properties) { @@ -1045,9 +1111,15 @@ const SchemaUtil::SchemaDelta SchemaUtil::ComputeCompatibilityDelta( // If we've gained any new joinable properties, then the joinable property // ids may change. Since the joinable property ids are stored in the cache, - // we'll need to reconstruct joinable cache. + // we'll need to reconstruct join index. + // If we've gained any new nested document properties, we also rebuild the + // join index. This is because we index all nested joinable properties, so + // adding a nested document property will most probably result in having + // more joinable properties. if (new_parsed_property_configs.num_joinable_properties > - old_joinable_properties) { + old_joinable_properties || + new_parsed_property_configs.num_nested_document_properties > + old_nested_document_properties) { ICING_VLOG(1) << "Set of joinable properties in schema type '" << old_type_config.schema_type() << "' has changed, required reconstructing joinable cache."; diff --git a/icing/schema/schema-util.h b/icing/schema/schema-util.h index e707758..6d0ff73 100644 --- a/icing/schema/schema-util.h +++ b/icing/schema/schema-util.h @@ -121,6 +121,9 @@ class SchemaUtil { // Total number of properties that have joinable config int32_t num_joinable_properties = 0; + + // Total number of properties that have DataType::DOCUMENT + int32_t num_nested_document_properties = 0; }; // This function validates: @@ -157,6 +160,9 @@ class SchemaUtil { // (property whose joinable config is not NONE), OR // ii. Any type node in the cycle has a nested-type (direct or // indirect) with a joinable property. + // 15. For DOCUMENT data types, if + // DocumentIndexingConfig.indexable_nested_properties_list is non-empty, + // DocumentIndexingConfig.index_nested_properties must be false. // // Returns: // On success, a dependent map from each types to their dependent types @@ -315,6 +321,17 @@ class SchemaUtil { PropertyConfigProto::Cardinality::Code cardinality, std::string_view schema_type, std::string_view property_name); + // Checks that the 'document_indexing_config' satisfies the following rule: + // 1. If indexable_nested_properties is non-empty, index_nested_properties + // must be set to false. + // + // Returns: + // INVALID_ARGUMENT if any of the rules are not followed + // OK on success + static libtextclassifier3::Status ValidateDocumentIndexingConfig( + const DocumentIndexingConfig& config, std::string_view schema_type, + std::string_view property_name); + // Returns if 'parent_type' is a direct or indirect parent of 'child_type'. static bool IsParent(const SchemaUtil::InheritanceMap& inheritance_map, std::string_view parent_type, diff --git a/icing/schema/schema-util_test.cc b/icing/schema/schema-util_test.cc index 40e30b0..564bbc0 100644 --- a/icing/schema/schema-util_test.cc +++ b/icing/schema/schema-util_test.cc @@ -14,6 +14,8 @@ #include "icing/schema/schema-util.h" +#include <initializer_list> +#include <string> #include <string_view> #include <unordered_set> @@ -2790,6 +2792,438 @@ TEST_P(SchemaUtilTest, IsEmpty()); } +TEST_P(SchemaUtilTest, + AddingNewIndexedDocumentPropertyMakesIndexAndJoinIncompatible) { + SchemaTypeConfigProto nested_schema = + SchemaTypeConfigBuilder() + .SetType(kEmailType) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + + // Configure old schema + SchemaProto old_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // Configure new schema + SchemaProto new_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("NewEmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_index_incompatible.insert(kPersonType); + schema_delta.schema_types_join_incompatible.insert(kPersonType); + + SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; + SchemaUtil::SchemaDelta result_schema_delta = + SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, + dependents_map); + EXPECT_THAT(result_schema_delta, Eq(schema_delta)); +} + +TEST_P( + SchemaUtilTest, + AddingNewIndexedDocumentPropertyWithIndexableListMakesIndexAndJoinIncompatible) { + SchemaTypeConfigProto nested_schema = + SchemaTypeConfigBuilder() + .SetType(kEmailType) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + + // Configure old schema + SchemaProto old_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // Configure new schema. The added nested document property is indexed, so + // this is both index and join incompatible + SchemaProto new_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType( + SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("NewEmailProperty") + .SetDataTypeDocument( + kEmailType, + /*indexable_nested_properties_list=*/ + std::initializer_list<std::string>{"subject"}) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_index_incompatible.insert(kPersonType); + schema_delta.schema_types_join_incompatible.insert(kPersonType); + + SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; + SchemaUtil::SchemaDelta result_schema_delta = + SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, + dependents_map); + EXPECT_THAT(result_schema_delta, Eq(schema_delta)); +} + +TEST_P(SchemaUtilTest, + AddingNewNonIndexedDocumentPropertyMakesJoinIncompatible) { + SchemaTypeConfigProto nested_schema = + SchemaTypeConfigBuilder() + .SetType(kEmailType) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + + // Configure old schema + SchemaProto old_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // Configure new schema. The added nested document property is not indexed, so + // this is index compatible, but join incompatible + SchemaProto new_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("NewEmailProperty") + .SetDataTypeDocument( + kEmailType, + /*index_nested_properties=*/false) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_join_incompatible.insert(kPersonType); + + SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; + SchemaUtil::SchemaDelta result_schema_delta = + SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, + dependents_map); + EXPECT_THAT(result_schema_delta, Eq(schema_delta)); +} + +TEST_P(SchemaUtilTest, DeletingIndexedDocumentPropertyIsIncompatible) { + SchemaTypeConfigProto nested_schema = + SchemaTypeConfigBuilder() + .SetType(kEmailType) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + + // Configure old schemam with two nested document properties of the same type + SchemaProto old_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("EmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("AnotherEmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // Configure new schema and drop one of the nested document properties + SchemaProto new_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("EmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_incompatible.insert(kPersonType); + schema_delta.schema_types_index_incompatible.insert(kPersonType); + schema_delta.schema_types_join_incompatible.insert(kPersonType); + + SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; + SchemaUtil::SchemaDelta result_schema_delta = + SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, + dependents_map); + EXPECT_THAT(result_schema_delta, Eq(schema_delta)); +} + +TEST_P(SchemaUtilTest, + DeletingNonIndexedDocumentPropertyIsIncompatible) { + SchemaTypeConfigProto nested_schema = + SchemaTypeConfigBuilder() + .SetType(kEmailType) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + + // Configure old schemam with two nested document properties of the same type + SchemaProto old_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("EmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("AnotherEmailProperty") + .SetDataTypeDocument( + kEmailType, + /*index_nested_properties=*/false) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // Configure new schema and drop the non-indexed nested document property + SchemaProto new_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("EmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_incompatible.insert(kPersonType); + schema_delta.schema_types_join_incompatible.insert(kPersonType); + + SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; + SchemaUtil::SchemaDelta result_schema_delta = + SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, + dependents_map); + EXPECT_THAT(result_schema_delta, Eq(schema_delta)); +} + +TEST_P(SchemaUtilTest, ChangingIndexedDocumentPropertyIsIncompatible) { + SchemaTypeConfigProto nested_schema = + SchemaTypeConfigBuilder() + .SetType(kEmailType) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + + // Configure old schemam with two nested document properties of the same type + SchemaProto old_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("EmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("AnotherEmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // Configure new schema and change one of the nested document properties + // to a different name (this is the same as deleting a property and adding + // another) + SchemaProto new_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("EmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("DifferentEmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_incompatible.insert(kPersonType); + schema_delta.schema_types_index_incompatible.insert(kPersonType); + schema_delta.schema_types_join_incompatible.insert(kPersonType); + + SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; + SchemaUtil::SchemaDelta result_schema_delta = + SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, + dependents_map); + EXPECT_THAT(result_schema_delta, Eq(schema_delta)); +} + +TEST_P(SchemaUtilTest, ChangingNonIndexedDocumentPropertyIsIncompatible) { + SchemaTypeConfigProto nested_schema = + SchemaTypeConfigBuilder() + .SetType(kEmailType) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + + // Configure old schemam with two nested document properties of the same type + SchemaProto old_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("EmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("AnotherEmailProperty") + .SetDataTypeDocument( + kEmailType, + /*index_nested_properties=*/false) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + // Configure new schema and change the non-indexed nested document property to + // a different name (this is the same as deleting a property and adding + // another) + SchemaProto new_schema = + SchemaBuilder() + .AddType(nested_schema) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("Property") + .SetDataTypeInt64(NUMERIC_MATCH_RANGE) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty( + PropertyConfigBuilder() + .SetName("EmailProperty") + .SetDataTypeDocument( + kEmailType, /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("DifferentEmailProperty") + .SetDataTypeDocument( + kEmailType, + /*index_nested_properties=*/false) + .SetCardinality(CARDINALITY_OPTIONAL))) + .Build(); + + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_incompatible.insert(kPersonType); + schema_delta.schema_types_join_incompatible.insert(kPersonType); + + SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; + SchemaUtil::SchemaDelta result_schema_delta = + SchemaUtil::ComputeCompatibilityDelta(old_schema, new_schema, + dependents_map); + EXPECT_THAT(result_schema_delta, Eq(schema_delta)); +} + TEST_P(SchemaUtilTest, ChangingJoinablePropertiesMakesJoinIncompatible) { // Configure old schema SchemaProto schema_with_joinable_property = @@ -3081,6 +3515,239 @@ TEST_P(SchemaUtilTest, IndexNestedDocumentsIndexIncompatible) { EXPECT_THAT(actual, Eq(schema_delta)); } +TEST_P(SchemaUtilTest, AddOrDropIndexableNestedProperties_IndexIncompatible) { + SchemaTypeConfigProto email_type_config = + SchemaTypeConfigBuilder() + .SetType(kEmailType) + .AddProperty(PropertyConfigBuilder() + .SetName("recipient") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("body") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + SchemaProto schema_1 = + SchemaBuilder() + .AddType(email_type_config) + .AddType( + SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("emails") + .SetDataTypeDocument( + kEmailType, + /*indexable_nested_properties_list=*/ + {"recipient", "subject", "body"}) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + SchemaProto schema_2 = + SchemaBuilder() + .AddType(email_type_config) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("emails") + .SetDataTypeDocument( + kEmailType, + /*indexable_nested_properties=*/ + {"recipient", "subject"}) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + // Dropping some indexable_nested_properties should make kPersonType + // index_incompatible. kEmailType should be unaffected. + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_index_incompatible.emplace(kPersonType); + SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; + SchemaUtil::SchemaDelta actual = + SchemaUtil::ComputeCompatibilityDelta(schema_1, schema_2, dependents_map); + EXPECT_THAT(actual, Eq(schema_delta)); + + // Adding some indexable_nested_properties should also make kPersonType + // index_incompatible. kEmailType should be unaffected. + actual = + SchemaUtil::ComputeCompatibilityDelta(schema_2, schema_1, dependents_map); + EXPECT_THAT(actual, Eq(schema_delta)); +} + +TEST_P(SchemaUtilTest, ChangingIndexableNestedProperties_IndexIncompatible) { + SchemaTypeConfigProto email_type_config = + SchemaTypeConfigBuilder() + .SetType(kEmailType) + .AddProperty(PropertyConfigBuilder() + .SetName("recipient") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("body") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + SchemaProto schema_1 = + SchemaBuilder() + .AddType(email_type_config) + .AddType( + SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("emails") + .SetDataTypeDocument( + kEmailType, + /*indexable_nested_properties_list=*/ + {"recipient", "subject"}) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + SchemaProto schema_2 = + SchemaBuilder() + .AddType(email_type_config) + .AddType( + SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("emails") + .SetDataTypeDocument( + kEmailType, + /*indexable_nested_properties_list=*/ + {"recipient", "body"}) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + // Changing 'subject' to 'body' for indexable_nested_properties_list should + // make kPersonType index_incompatible. kEmailType should be unaffected. + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_index_incompatible.emplace(kPersonType); + SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; + SchemaUtil::SchemaDelta actual = + SchemaUtil::ComputeCompatibilityDelta(schema_1, schema_2, dependents_map); + EXPECT_THAT(actual, Eq(schema_delta)); +} + +TEST_P(SchemaUtilTest, IndexableNestedPropertiesFullSet_IndexIncompatible) { + SchemaTypeConfigProto email_type_config = + SchemaTypeConfigBuilder() + .SetType(kEmailType) + .AddProperty(PropertyConfigBuilder() + .SetName("recipient") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("body") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + SchemaProto schema_1 = + SchemaBuilder() + .AddType(email_type_config) + .AddType(SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("emails") + .SetDataTypeDocument( + kEmailType, + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + SchemaProto schema_2 = + SchemaBuilder() + .AddType(email_type_config) + .AddType( + SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("emails") + .SetDataTypeDocument( + kEmailType, + /*indexable_nested_properties_list=*/ + {"recipient", "body", "subject"}) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + // This scenario also invalidates kPersonType and triggers an index rebuild at + // the moment, even though the set of indexable_nested_properties from + // schema_1 to schema_2 should be the same. + SchemaUtil::SchemaDelta schema_delta; + schema_delta.schema_types_index_incompatible.emplace(kPersonType); + SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; + SchemaUtil::SchemaDelta actual = + SchemaUtil::ComputeCompatibilityDelta(schema_1, schema_2, dependents_map); + EXPECT_THAT(actual, Eq(schema_delta)); +} + +TEST_P(SchemaUtilTest, + ChangingIndexableNestedPropertiesOrder_IndexIsCompatible) { + SchemaTypeConfigProto email_type_config = + SchemaTypeConfigBuilder() + .SetType(kEmailType) + .AddProperty(PropertyConfigBuilder() + .SetName("recipient") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("subject") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("body") + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .Build(); + SchemaProto schema_1 = + SchemaBuilder() + .AddType(email_type_config) + .AddType( + SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("emails") + .SetDataTypeDocument( + kEmailType, + /*indexable_nested_properties_list=*/ + {"recipient", "subject", "body"}) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + SchemaProto schema_2 = + SchemaBuilder() + .AddType(email_type_config) + .AddType( + SchemaTypeConfigBuilder() + .SetType(kPersonType) + .AddProperty(PropertyConfigBuilder() + .SetName("emails") + .SetDataTypeDocument( + kEmailType, + /*indexable_nested_properties_list=*/ + {"subject", "body", "recipient"}) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + // Changing order of elements in indexable_nested_properties_list should have + // no effect on schema compatibility. + SchemaUtil::SchemaDelta schema_delta; + SchemaUtil::DependentMap dependents_map = {{kEmailType, {{kPersonType, {}}}}}; + SchemaUtil::SchemaDelta actual = + SchemaUtil::ComputeCompatibilityDelta(schema_1, schema_2, dependents_map); + EXPECT_THAT(actual, Eq(schema_delta)); + EXPECT_THAT(actual.schema_types_index_incompatible, IsEmpty()); +} + TEST_P(SchemaUtilTest, ValidateStringIndexingConfigShouldHaveTermMatchType) { SchemaProto schema = SchemaBuilder() @@ -3673,6 +4340,137 @@ TEST_P(SchemaUtilTest, ValidateNestedJoinablePropertyDiamondRelationship) { StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } +TEST_P(SchemaUtilTest, + ValidDocumentIndexingConfigFields_emptyIndexableListBooleanTrue) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("InnerSchema") + .AddProperty(PropertyConfigBuilder() + .SetName("prop1") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("prop2") + .SetDataTypeString(TERM_MATCH_UNKNOWN, + TOKENIZER_NONE) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("OuterSchema") + .AddProperty(PropertyConfigBuilder() + .SetName("InnerProperty") + .SetDataTypeDocument( + "InnerSchema", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + SchemaTypeConfigProto* outerSchemaType = schema.mutable_types(1); + outerSchemaType->mutable_properties(0) + ->mutable_document_indexing_config() + ->clear_indexable_nested_properties_list(); + + EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); +} + +TEST_P(SchemaUtilTest, + ValidDocumentIndexingConfigFields_emptyIndexableListBooleanFalse) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("InnerSchema") + .AddProperty(PropertyConfigBuilder() + .SetName("prop1") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL)) + .AddProperty(PropertyConfigBuilder() + .SetName("prop2") + .SetDataTypeString(TERM_MATCH_UNKNOWN, + TOKENIZER_NONE) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("OuterSchema") + .AddProperty(PropertyConfigBuilder() + .SetName("InnerProperty") + .SetDataTypeDocument( + "InnerSchema", + /*index_nested_properties=*/false) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + SchemaTypeConfigProto* outerSchemaType = schema.mutable_types(1); + outerSchemaType->mutable_properties(0) + ->mutable_document_indexing_config() + ->clear_indexable_nested_properties_list(); + + EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); +} + +TEST_P(SchemaUtilTest, + ValidDocumentIndexingConfigFields_nonEmptyIndexableList) { + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("InnerSchema") + .AddProperty(PropertyConfigBuilder() + .SetName("prop1") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("OuterSchema") + .AddProperty( + PropertyConfigBuilder() + .SetName("InnerProperty") + .SetDataTypeDocument( + "InnerSchema", + /*indexable_nested_properties_list=*/ + std::initializer_list<std::string>{"prop1"}) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + SchemaTypeConfigProto* outerSchemaType = schema.mutable_types(1); + outerSchemaType->mutable_properties(0) + ->mutable_document_indexing_config() + ->set_index_nested_properties(false); + EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), IsOk()); +} + +TEST_P(SchemaUtilTest, InvalidDocumentIndexingConfigFields) { + // If indexable_nested_properties is non-empty, index_nested_properties is + // required to be false. + SchemaProto schema = + SchemaBuilder() + .AddType(SchemaTypeConfigBuilder() + .SetType("InnerSchema") + .AddProperty(PropertyConfigBuilder() + .SetName("prop1") + .SetDataTypeString(TERM_MATCH_PREFIX, + TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL))) + .AddType(SchemaTypeConfigBuilder() + .SetType("OuterSchema") + .AddProperty(PropertyConfigBuilder() + .SetName("InnerProperty") + .SetDataTypeDocument( + "InnerSchema", + /*index_nested_properties=*/true) + .SetCardinality(CARDINALITY_REPEATED))) + .Build(); + + // Setting a non-empty indexable_nested_properties_list while + // index_nested_properties=true is invalid. + SchemaTypeConfigProto* outerSchemaType = schema.mutable_types(1); + outerSchemaType->mutable_properties(0) + ->mutable_document_indexing_config() + ->add_indexable_nested_properties_list("prop"); + + EXPECT_THAT(SchemaUtil::Validate(schema, GetParam()), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); +} + TEST_P(SchemaUtilTest, MultipleReferencesToSameNestedSchemaOk) { SchemaProto schema = SchemaBuilder() diff --git a/icing/schema/section-manager-builder_test.cc b/icing/schema/section-manager-builder_test.cc index 60dd507..1d452d5 100644 --- a/icing/schema/section-manager-builder_test.cc +++ b/icing/schema/section-manager-builder_test.cc @@ -270,9 +270,11 @@ TEST_P(NonIndexableSectionManagerBuilderTest, Build) { ICING_ASSERT_OK(builder.ProcessSchemaTypePropertyConfig( /*schema_type_id=*/0, property_config, std::string(kPropertyPath))); + // NonIndexable sections will still consume a sectionId. std::unique_ptr<SectionManager> section_manager = std::move(builder).Build(); EXPECT_THAT(section_manager->GetMetadataList(std::string(kSchemaType)), - IsOkAndHolds(Pointee(IsEmpty()))); + IsOkAndHolds(Pointee(ElementsAre(EqualsSectionMetadata( + /*expected_id=*/0, kPropertyPath, property_config))))); } // The following types are considered non-indexable: diff --git a/icing/schema/section-manager.cc b/icing/schema/section-manager.cc index 38042d0..3d540d6 100644 --- a/icing/schema/section-manager.cc +++ b/icing/schema/section-manager.cc @@ -15,15 +15,9 @@ #include "icing/schema/section-manager.h" #include <algorithm> -#include <cinttypes> -#include <cstddef> #include <cstdint> -#include <iterator> -#include <memory> #include <string> #include <string_view> -#include <unordered_map> -#include <unordered_set> #include <utility> #include <vector> @@ -35,7 +29,6 @@ #include "icing/proto/schema.pb.h" #include "icing/proto/term.pb.h" #include "icing/schema/property-util.h" -#include "icing/schema/schema-util.h" #include "icing/schema/section.h" #include "icing/store/document-filter-data.h" #include "icing/store/key-mapper.h" @@ -99,12 +92,14 @@ SectionManager::Builder::ProcessSchemaTypePropertyConfig( return absl_ports::InvalidArgumentError("Invalid schema type id"); } - if (SchemaUtil::IsIndexedProperty(property_config)) { - ICING_RETURN_IF_ERROR( - AppendNewSectionMetadata(§ion_metadata_cache_[schema_type_id], - std::move(property_path), property_config)); - } - + // We don't need to check if the property is indexable. This method will + // only be called properties that should consume sectionIds, even if the + // property's indexing configuration itself is not indexable. + // This would be the case for unknown and non-indexable property paths that + // are defined in the indexable_nested_properties_list. + ICING_RETURN_IF_ERROR( + AppendNewSectionMetadata(§ion_metadata_cache_[schema_type_id], + std::move(property_path), property_config)); return libtextclassifier3::Status::OK; } @@ -141,6 +136,13 @@ libtextclassifier3::StatusOr<SectionGroup> SectionManager::ExtractSections( for (const SectionMetadata& section_metadata : *metadata_list) { switch (section_metadata.data_type) { case PropertyConfigProto::DataType::STRING: { + if (section_metadata.term_match_type == TermMatchType::UNKNOWN || + section_metadata.tokenizer == + StringIndexingConfig::TokenizerType::NONE) { + // Skip if term-match type is UNKNOWN, or if the tokenizer-type is + // NONE. + break; + } AppendSection( section_metadata, property_util::ExtractPropertyValuesFromDocument<std::string_view>( @@ -149,6 +151,11 @@ libtextclassifier3::StatusOr<SectionGroup> SectionManager::ExtractSections( break; } case PropertyConfigProto::DataType::INT64: { + if (section_metadata.numeric_match_type == + IntegerIndexingConfig::NumericMatchType::UNKNOWN) { + // Skip if numeric-match type is UNKNOWN. + break; + } AppendSection(section_metadata, property_util::ExtractPropertyValuesFromDocument<int64_t>( document, section_metadata.path), diff --git a/icing/schema/section-manager_test.cc b/icing/schema/section-manager_test.cc index db2be6b..eee78e9 100644 --- a/icing/schema/section-manager_test.cc +++ b/icing/schema/section-manager_test.cc @@ -14,7 +14,6 @@ #include "icing/schema/section-manager.h" -#include <limits> #include <memory> #include <string> #include <string_view> @@ -25,7 +24,6 @@ #include "icing/file/filesystem.h" #include "icing/proto/document.pb.h" #include "icing/proto/schema.pb.h" -#include "icing/proto/term.pb.h" #include "icing/schema-builder.h" #include "icing/schema/schema-type-manager.h" #include "icing/schema/schema-util.h" @@ -63,6 +61,28 @@ static constexpr std::string_view kTypeConversation = "Conversation"; static constexpr std::string_view kPropertyEmails = "emails"; static constexpr std::string_view kPropertyName = "name"; +// type and property names of Group +static constexpr std::string_view kTypeGroup = "Group"; +// indexable +static constexpr std::string_view kPropertyConversation = "conversation"; +static constexpr std::string_view kPropertyGroupName = "groupName"; +// nested indexable +static constexpr std::string_view kPropertyNestedConversationName = "name"; +static constexpr std::string_view kPropertyNestedConversationEmailRecipientIds = + "emails.recipientIds"; +static constexpr std::string_view kPropertyNestedConversationEmailRecipient = + "emails.recipients"; +static constexpr std::string_view kPropertyNestedConversationEmailSubject = + "emails.subject"; +// nested non-indexable +static constexpr std::string_view kPropertyNestedConversationEmailAttachment = + "emails.attachment"; +// non-existent property path +static constexpr std::string_view kPropertyNestedNonExistent = + "emails.nonExistentNestedProperty"; +static constexpr std::string_view kPropertyNestedNonExistent2 = + "emails.nonExistentNestedProperty2"; + constexpr int64_t kDefaultTimestamp = 1663274901; PropertyConfigProto CreateRecipientIdsPropertyConfig() { @@ -105,6 +125,22 @@ PropertyConfigProto CreateNamePropertyConfig() { .Build(); } +PropertyConfigProto CreateAttachmentPropertyConfig() { + return PropertyConfigBuilder() + .SetName(kPropertyAttachment) + .SetDataType(TYPE_BYTES) + .SetCardinality(CARDINALITY_OPTIONAL) + .Build(); +} + +PropertyConfigProto CreateGroupNamePropertyConfig() { + return PropertyConfigBuilder() + .SetName(kPropertyGroupName) + .SetDataTypeString(TERM_MATCH_EXACT, TOKENIZER_PLAIN) + .SetCardinality(CARDINALITY_OPTIONAL) + .Build(); +} + SchemaTypeConfigProto CreateEmailTypeConfig() { return SchemaTypeConfigBuilder() .SetType(kTypeEmail) @@ -139,6 +175,28 @@ SchemaTypeConfigProto CreateConversationTypeConfig() { .Build(); } +SchemaTypeConfigProto CreateGroupTypeConfig() { + return SchemaTypeConfigBuilder() + .SetType(kTypeGroup) + .AddProperty(CreateGroupNamePropertyConfig()) + .AddProperty( + PropertyConfigBuilder() + .SetName(kPropertyConversation) + .SetDataTypeDocument( + kTypeConversation, + /*indexable_nested_properties_list=*/ + {std::string(kPropertyNestedConversationName), + std::string(kPropertyNestedConversationEmailRecipientIds), + std::string(kPropertyNestedConversationEmailSubject), + std::string(kPropertyNestedConversationEmailRecipient), + std::string(kPropertyNestedConversationEmailAttachment), + std::string(kPropertyNestedNonExistent2), + std::string(kPropertyNestedNonExistent), + std::string(kPropertyNestedNonExistent)}) + .SetCardinality(CARDINALITY_REPEATED)) + .Build(); +} + class SectionManagerTest : public ::testing::Test { protected: void SetUp() override { @@ -146,9 +204,11 @@ class SectionManagerTest : public ::testing::Test { auto email_type = CreateEmailTypeConfig(); auto conversation_type = CreateConversationTypeConfig(); + auto group_type = CreateGroupTypeConfig(); type_config_map_.emplace(email_type.schema_type(), email_type); type_config_map_.emplace(conversation_type.schema_type(), conversation_type); + type_config_map_.emplace(group_type.schema_type(), group_type); // DynamicTrieKeyMapper uses 3 internal arrays for bookkeeping. Give each // one 128KiB so the total DynamicTrieKeyMapper should get 384KiB @@ -158,6 +218,7 @@ class SectionManagerTest : public ::testing::Test { filesystem_, test_dir_, key_mapper_size)); ICING_ASSERT_OK(schema_type_mapper_->Put(kTypeEmail, 0)); ICING_ASSERT_OK(schema_type_mapper_->Put(kTypeConversation, 1)); + ICING_ASSERT_OK(schema_type_mapper_->Put(kTypeGroup, 2)); email_document_ = DocumentBuilder() @@ -183,6 +244,15 @@ class SectionManagerTest : public ::testing::Test { DocumentProto(email_document_), DocumentProto(email_document_)) .Build(); + + group_document_ = + DocumentBuilder() + .SetKey("icing", "group/1") + .SetSchema(std::string(kTypeGroup)) + .AddDocumentProperty(std::string(kPropertyConversation), + DocumentProto(conversation_document_)) + .AddStringProperty(std::string(kPropertyGroupName), "group_name_1") + .Build(); } void TearDown() override { @@ -197,6 +267,7 @@ class SectionManagerTest : public ::testing::Test { DocumentProto email_document_; DocumentProto conversation_document_; + DocumentProto group_document_; }; TEST_F(SectionManagerTest, ExtractSections) { @@ -295,6 +366,91 @@ TEST_F(SectionManagerTest, ExtractSectionsNested) { ElementsAre(kDefaultTimestamp, kDefaultTimestamp)); } +TEST_F(SectionManagerTest, ExtractSectionsIndexableNestedPropertiesList) { + // Use SchemaTypeManager factory method to instantiate SectionManager. + ICING_ASSERT_OK_AND_ASSIGN( + std::unique_ptr<SchemaTypeManager> schema_type_manager, + SchemaTypeManager::Create(type_config_map_, schema_type_mapper_.get())); + + // Extracts all sections from 'Group' document + ICING_ASSERT_OK_AND_ASSIGN( + SectionGroup section_group, + schema_type_manager->section_manager().ExtractSections(group_document_)); + + // SectionId assignments: + // 0 -> conversation.emails.attachment (bytes, non-indexable) + // 1 -> conversation.emails.recipientIds (int64) + // 2 -> conversation.emails.recipients (string) + // 3 -> conversation.emails.subject (string) + // 4 -> conversation.name + // (string, but no entry for this in conversation_document_) + // 5 -> groupName (string) + // 6 -> conversation.emails.nonExistentNestedProperty + // (unknown, non-indexable) + // 7 -> conversation.emails.nonExistentNestedProperty2 + // (unknown, non-indexable) + // + // SectionId assignment order: + // - We assign section ids to known (existing) properties first in alphabet + // order. + // - After handling all known properties, we assign section ids to all unknown + // (non-existent) properties that are specified in the + // indexable_nested_properties_list. + // - As a result, assignment of the entire section set is not done + // alphabetically, but assignment is still deterministic and alphabetical + // order is preserved inside the known properties and unknown properties + // sets individually. + // + // 'conversation.emails.attachment', + // 'conversation.emails.nonExistentNestedProperty' and + // 'conversation.emails.nonExistentNestedProperty2' are assigned sectionIds + // even though they are non-indexable because they appear in 'Group' schema + // type's indexable_nested_props_list. + // However 'conversation.emails.attachment' does not exist in section_group + // (even though the property exists and has a sectionId assignment) as + // SectionManager::ExtractSections only extracts indexable string and integer + // section data from a document. + + // String sections + EXPECT_THAT(section_group.string_sections, SizeIs(3)); + + EXPECT_THAT(section_group.string_sections[0].metadata, + EqualsSectionMetadata( + /*expected_id=*/2, + /*expected_property_path=*/"conversation.emails.recipients", + CreateRecipientsPropertyConfig())); + EXPECT_THAT(section_group.string_sections[0].content, + ElementsAre("recipient1", "recipient2", "recipient3", + "recipient1", "recipient2", "recipient3")); + + EXPECT_THAT(section_group.string_sections[1].metadata, + EqualsSectionMetadata( + /*expected_id=*/3, + /*expected_property_path=*/"conversation.emails.subject", + CreateSubjectPropertyConfig())); + EXPECT_THAT(section_group.string_sections[1].content, + ElementsAre("the subject", "the subject")); + + EXPECT_THAT(section_group.string_sections[2].metadata, + EqualsSectionMetadata( + /*expected_id=*/5, + /*expected_property_path=*/"groupName", + CreateGroupNamePropertyConfig())); + EXPECT_THAT(section_group.string_sections[2].content, + ElementsAre("group_name_1")); + + // Integer sections + EXPECT_THAT(section_group.integer_sections, SizeIs(1)); + + EXPECT_THAT(section_group.integer_sections[0].metadata, + EqualsSectionMetadata( + /*expected_id=*/1, + /*expected_property_path=*/"conversation.emails.recipientIds", + CreateRecipientIdsPropertyConfig())); + EXPECT_THAT(section_group.integer_sections[0].content, + ElementsAre(1, 2, 3, 1, 2, 3)); +} + TEST_F(SectionManagerTest, GetSectionMetadata) { // Use SchemaTypeManager factory method to instantiate SectionManager. ICING_ASSERT_OK_AND_ASSIGN( @@ -352,6 +508,86 @@ TEST_F(SectionManagerTest, GetSectionMetadata) { IsOkAndHolds(Pointee(EqualsSectionMetadata( /*expected_id=*/4, /*expected_property_path=*/"name", CreateNamePropertyConfig())))); + + // Group (section id -> section property path): + // 0 -> conversation.emails.attachment (non-indexable) + // 1 -> conversation.emails.recipientIds + // 2 -> conversation.emails.recipients + // 3 -> conversation.emails.subject + // 4 -> conversation.name + // 5 -> groupName + // 6 -> conversation.emails.nonExistentNestedProperty (non-indexable) + // 7 -> conversation.emails.nonExistentNestedProperty2 (non-indexable) + // + // SectionId assignment order: + // - We assign section ids to known (existing) properties first in alphabet + // order. + // - After handling all known properties, we assign section ids to all unknown + // (non-existent) properties that are specified in the + // indexable_nested_properties_list. + // - As a result, assignment of the entire section set is not done + // alphabetically, but assignment is still deterministic and alphabetical + // order is preserved inside the known properties and unknown properties + // sets individually. + EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( + /*schema_type_id=*/2, /*section_id=*/0), + IsOkAndHolds(Pointee(EqualsSectionMetadata( + /*expected_id=*/0, + /*expected_property_path=*/"conversation.emails.attachment", + CreateAttachmentPropertyConfig())))); + EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( + /*schema_type_id=*/2, /*section_id=*/1), + IsOkAndHolds(Pointee(EqualsSectionMetadata( + /*expected_id=*/1, + /*expected_property_path=*/"conversation.emails.recipientIds", + CreateRecipientIdsPropertyConfig())))); + EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( + /*schema_type_id=*/2, /*section_id=*/2), + IsOkAndHolds(Pointee(EqualsSectionMetadata( + /*expected_id=*/2, + /*expected_property_path=*/"conversation.emails.recipients", + CreateRecipientsPropertyConfig())))); + EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( + /*schema_type_id=*/2, /*section_id=*/3), + IsOkAndHolds(Pointee(EqualsSectionMetadata( + /*expected_id=*/3, + /*expected_property_path=*/"conversation.emails.subject", + CreateSubjectPropertyConfig())))); + EXPECT_THAT( + schema_type_manager->section_manager().GetSectionMetadata( + /*schema_type_id=*/2, /*section_id=*/4), + IsOkAndHolds(Pointee(EqualsSectionMetadata( + /*expected_id=*/4, /*expected_property_path=*/"conversation.name", + CreateNamePropertyConfig())))); + EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( + /*schema_type_id=*/2, /*section_id=*/5), + IsOkAndHolds(Pointee(EqualsSectionMetadata( + /*expected_id=*/5, /*expected_property_path=*/"groupName", + CreateGroupNamePropertyConfig())))); + EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( + /*schema_type_id=*/2, /*section_id=*/6), + IsOkAndHolds(Pointee(EqualsSectionMetadata( + /*expected_id=*/6, + /*expected_property_path=*/ + "conversation.emails.nonExistentNestedProperty", + PropertyConfigBuilder() + .SetName("nonExistentNestedProperty") + .SetDataType(TYPE_UNKNOWN) + .Build())))); + EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( + /*schema_type_id=*/2, /*section_id=*/7), + IsOkAndHolds(Pointee(EqualsSectionMetadata( + /*expected_id=*/7, + /*expected_property_path=*/ + "conversation.emails.nonExistentNestedProperty2", + PropertyConfigBuilder() + .SetName("nonExistentNestedProperty2") + .SetDataType(TYPE_UNKNOWN) + .Build())))); + // Check that no more properties are indexed + EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( + /*schema_type_id=*/2, /*section_id=*/8), + StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } TEST_F(SectionManagerTest, GetSectionMetadataInvalidSchemaTypeId) { @@ -359,13 +595,13 @@ TEST_F(SectionManagerTest, GetSectionMetadataInvalidSchemaTypeId) { ICING_ASSERT_OK_AND_ASSIGN( std::unique_ptr<SchemaTypeManager> schema_type_manager, SchemaTypeManager::Create(type_config_map_, schema_type_mapper_.get())); - ASSERT_THAT(type_config_map_, SizeIs(2)); + ASSERT_THAT(type_config_map_, SizeIs(3)); EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( /*schema_type_id=*/-1, /*section_id=*/0), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); EXPECT_THAT(schema_type_manager->section_manager().GetSectionMetadata( - /*schema_type_id=*/2, /*section_id=*/0), + /*schema_type_id=*/3, /*section_id=*/0), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT)); } diff --git a/icing/scoring/advanced_scoring/advanced-scorer_fuzz_test.cc b/icing/scoring/advanced_scoring/advanced-scorer_fuzz_test.cc index bdafa28..3612359 100644 --- a/icing/scoring/advanced_scoring/advanced-scorer_fuzz_test.cc +++ b/icing/scoring/advanced_scoring/advanced-scorer_fuzz_test.cc @@ -37,13 +37,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { SchemaStore::Create(&filesystem, schema_store_dir, &fake_clock) .ValueOrDie(); std::unique_ptr<DocumentStore> document_store = - DocumentStore::Create(&filesystem, doc_store_dir, &fake_clock, - schema_store.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr) + DocumentStore::Create( + &filesystem, doc_store_dir, &fake_clock, schema_store.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr) .ValueOrDie() .document_store; diff --git a/icing/scoring/advanced_scoring/advanced-scorer_test.cc b/icing/scoring/advanced_scoring/advanced-scorer_test.cc index 0ecc21d..cc1d413 100644 --- a/icing/scoring/advanced_scoring/advanced-scorer_test.cc +++ b/icing/scoring/advanced_scoring/advanced-scorer_test.cc @@ -64,13 +64,14 @@ class AdvancedScorerTest : public testing::Test { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, doc_store_dir_, &fake_clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); // Creates a simple email schema diff --git a/icing/scoring/score-and-rank_benchmark.cc b/icing/scoring/score-and-rank_benchmark.cc index abb019f..7cb5a95 100644 --- a/icing/scoring/score-and-rank_benchmark.cc +++ b/icing/scoring/score-and-rank_benchmark.cc @@ -95,7 +95,8 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( return DocumentStore::Create( filesystem, base_dir, clock, schema_store, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, /*initialize_stats=*/nullptr); } diff --git a/icing/scoring/scorer_test.cc b/icing/scoring/scorer_test.cc index 4a97a87..5194c7f 100644 --- a/icing/scoring/scorer_test.cc +++ b/icing/scoring/scorer_test.cc @@ -64,13 +64,14 @@ class ScorerTest : public ::testing::TestWithParam<ScorerTestingMode> { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock1_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, doc_store_dir_, &fake_clock1_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); // Creates a simple email schema diff --git a/icing/scoring/scoring-processor_test.cc b/icing/scoring/scoring-processor_test.cc index 644e013..deddff8 100644 --- a/icing/scoring/scoring-processor_test.cc +++ b/icing/scoring/scoring-processor_test.cc @@ -62,13 +62,14 @@ class ScoringProcessorTest ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, doc_store_dir_, &fake_clock_, - schema_store_.get(), - /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, doc_store_dir_, &fake_clock_, schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); document_store_ = std::move(create_result.document_store); // Creates a simple email schema diff --git a/icing/store/document-store.cc b/icing/store/document-store.cc index e99bacf..30de410 100644 --- a/icing/store/document-store.cc +++ b/icing/store/document-store.cc @@ -54,6 +54,7 @@ #include "icing/store/document-log-creator.h" #include "icing/store/dynamic-trie-key-mapper.h" #include "icing/store/namespace-id.h" +#include "icing/store/persistent-hash-map-key-mapper.h" #include "icing/store/usage-store.h" #include "icing/tokenization/language-segmenter.h" #include "icing/util/clock.h" @@ -73,6 +74,7 @@ namespace { // Used in DocumentId mapper to mark a document as deleted constexpr int64_t kDocDeletedFlag = -1; constexpr char kDocumentIdMapperFilename[] = "document_id_mapper"; +constexpr char kUriHashMapperWorkingPath[] = "uri_mapper"; constexpr char kDocumentStoreHeaderFilename[] = "document_store_header"; constexpr char kScoreCacheFilename[] = "score_cache"; constexpr char kCorpusScoreCache[] = "corpus_score_cache"; @@ -81,9 +83,17 @@ constexpr char kNamespaceMapperFilename[] = "namespace_mapper"; constexpr char kUsageStoreDirectoryName[] = "usage_store"; constexpr char kCorpusIdMapperFilename[] = "corpus_mapper"; -// Determined through manual testing to allow for 1 million uris. 1 million -// because we allow up to 1 million DocumentIds. -constexpr int32_t kUriMapperMaxSize = 36 * 1024 * 1024; // 36 MiB +// Determined through manual testing to allow for 4 million uris. 4 million +// because we allow up to 4 million DocumentIds. +constexpr int32_t kUriDynamicTrieKeyMapperMaxSize = + 144 * 1024 * 1024; // 144 MiB + +constexpr int32_t kUriHashKeyMapperMaxNumEntries = + kMaxDocumentId + 1; // 1 << 22, 4M +// - Key: namespace_id_str (3 bytes) + fingerprinted_uri (10 bytes) + '\0' (1 +// byte) +// - Value: DocumentId (4 bytes) +constexpr int32_t kUriHashKeyMapperKVByteSize = 13 + 1 + sizeof(DocumentId); // 384 KiB for a DynamicTrieKeyMapper would allow each internal array to have a // max of 128 KiB for storage. @@ -100,6 +110,10 @@ std::string MakeHeaderFilename(const std::string& base_dir) { return absl_ports::StrCat(base_dir, "/", kDocumentStoreHeaderFilename); } +std::string MakeUriHashMapperWorkingPath(const std::string& base_dir) { + return absl_ports::StrCat(base_dir, "/", kUriHashMapperWorkingPath); +} + std::string MakeDocumentIdMapperFilename(const std::string& base_dir) { return absl_ports::StrCat(base_dir, "/", kDocumentIdMapperFilename); } @@ -207,6 +221,41 @@ std::unordered_map<NamespaceId, std::string> GetNamespaceIdsToNamespaces( return namespace_ids_to_namespaces; } +libtextclassifier3::StatusOr<std::unique_ptr< + KeyMapper<DocumentId, fingerprint_util::FingerprintStringFormatter>>> +CreateUriMapper(const Filesystem& filesystem, const std::string& base_dir, + bool pre_mapping_fbv, bool use_persistent_hash_map) { + std::string uri_hash_mapper_working_path = + MakeUriHashMapperWorkingPath(base_dir); + // Due to historic issue, we use document store's base_dir directly as + // DynamicTrieKeyMapper's working directory for uri mapper. + // DynamicTrieKeyMapper also creates a subdirectory "key_mapper_dir", so the + // actual files will be put under "<base_dir>/key_mapper_dir/". + bool dynamic_trie_key_mapper_dir_exists = filesystem.DirectoryExists( + absl_ports::StrCat(base_dir, "/key_mapper_dir").c_str()); + bool persistent_hash_map_dir_exists = + filesystem.DirectoryExists(uri_hash_mapper_working_path.c_str()); + if ((use_persistent_hash_map && dynamic_trie_key_mapper_dir_exists) || + (!use_persistent_hash_map && persistent_hash_map_dir_exists)) { + // Return a failure here so that the caller can properly delete and rebuild + // this component. + return absl_ports::FailedPreconditionError("Key mapper type mismatch"); + } + + if (use_persistent_hash_map) { + return PersistentHashMapKeyMapper< + DocumentId, fingerprint_util::FingerprintStringFormatter>:: + Create(filesystem, std::move(uri_hash_mapper_working_path), + pre_mapping_fbv, + /*max_num_entries=*/kUriHashKeyMapperMaxNumEntries, + /*average_kv_byte_size=*/kUriHashKeyMapperKVByteSize); + } else { + return DynamicTrieKeyMapper<DocumentId, + fingerprint_util::FingerprintStringFormatter>:: + Create(filesystem, base_dir, kUriDynamicTrieKeyMapperMaxSize); + } +} + } // namespace std::string DocumentStore::MakeFingerprint( @@ -231,6 +280,7 @@ DocumentStore::DocumentStore(const Filesystem* filesystem, const Clock* clock, const SchemaStore* schema_store, bool namespace_id_fingerprint, + bool pre_mapping_fbv, bool use_persistent_hash_map, int32_t compression_level) : filesystem_(filesystem), base_dir_(base_dir), @@ -238,6 +288,8 @@ DocumentStore::DocumentStore(const Filesystem* filesystem, schema_store_(schema_store), document_validator_(schema_store), namespace_id_fingerprint_(namespace_id_fingerprint), + pre_mapping_fbv_(pre_mapping_fbv), + use_persistent_hash_map_(use_persistent_hash_map), compression_level_(compression_level) {} libtextclassifier3::StatusOr<DocumentId> DocumentStore::Put( @@ -266,6 +318,7 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> DocumentStore::Create( const Filesystem* filesystem, const std::string& base_dir, const Clock* clock, const SchemaStore* schema_store, bool force_recovery_and_revalidate_documents, bool namespace_id_fingerprint, + bool pre_mapping_fbv, bool use_persistent_hash_map, int32_t compression_level, InitializeStatsProto* initialize_stats) { ICING_RETURN_ERROR_IF_NULL(filesystem); ICING_RETURN_ERROR_IF_NULL(clock); @@ -273,7 +326,7 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> DocumentStore::Create( auto document_store = std::unique_ptr<DocumentStore>(new DocumentStore( filesystem, base_dir, clock, schema_store, namespace_id_fingerprint, - compression_level)); + pre_mapping_fbv, use_persistent_hash_map, compression_level)); ICING_ASSIGN_OR_RETURN( DataLoss data_loss, document_store->Initialize(force_recovery_and_revalidate_documents, @@ -293,9 +346,12 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> DocumentStore::Create( return absl_ports::InternalError("Couldn't delete header file"); } - // Document key mapper + // Document key mapper. Doesn't hurt to delete both dynamic trie and + // persistent hash map without checking. ICING_RETURN_IF_ERROR( DynamicTrieKeyMapper<DocumentId>::Delete(*filesystem, base_dir)); + ICING_RETURN_IF_ERROR(PersistentHashMapKeyMapper<DocumentId>::Delete( + *filesystem, MakeUriHashMapperWorkingPath(base_dir))); // Document id mapper ICING_RETURN_IF_ERROR(FileBackedVector<int64_t>::Delete( @@ -429,11 +485,8 @@ libtextclassifier3::Status DocumentStore::InitializeExistingDerivedFiles() { // TODO(b/144458732): Implement a more robust version of TC_ASSIGN_OR_RETURN // that can support error logging. - auto document_key_mapper_or = DynamicTrieKeyMapper< - DocumentId, - fingerprint_util::FingerprintStringFormatter>::Create(*filesystem_, - base_dir_, - kUriMapperMaxSize); + auto document_key_mapper_or = CreateUriMapper( + *filesystem_, base_dir_, pre_mapping_fbv_, use_persistent_hash_map_); if (!document_key_mapper_or.ok()) { ICING_LOG(ERROR) << document_key_mapper_or.status().error_message() << "Failed to initialize KeyMapper"; @@ -646,6 +699,10 @@ libtextclassifier3::Status DocumentStore::RegenerateDerivedFiles( } libtextclassifier3::Status DocumentStore::ResetDocumentKeyMapper() { + // Only one type of KeyMapper (either DynamicTrieKeyMapper or + // PersistentHashMapKeyMapper) will actually exist at any moment, but it is ok + // to call Delete() for both since Delete() returns OK if any of them doesn't + // exist. // TODO(b/139734457): Replace ptr.reset()->Delete->Create flow with Reset(). document_key_mapper_.reset(); // TODO(b/216487496): Implement a more robust version of TC_RETURN_IF_ERROR @@ -654,17 +711,21 @@ libtextclassifier3::Status DocumentStore::ResetDocumentKeyMapper() { DynamicTrieKeyMapper<DocumentId>::Delete(*filesystem_, base_dir_); if (!status.ok()) { ICING_LOG(ERROR) << status.error_message() - << "Failed to delete old key mapper"; + << "Failed to delete old dynamic trie key mapper"; + return status; + } + status = PersistentHashMapKeyMapper<DocumentId>::Delete( + *filesystem_, MakeUriHashMapperWorkingPath(base_dir_)); + if (!status.ok()) { + ICING_LOG(ERROR) << status.error_message() + << "Failed to delete old persistent hash map key mapper"; return status; } // TODO(b/216487496): Implement a more robust version of TC_ASSIGN_OR_RETURN // that can support error logging. - auto document_key_mapper_or = DynamicTrieKeyMapper< - DocumentId, - fingerprint_util::FingerprintStringFormatter>::Create(*filesystem_, - base_dir_, - kUriMapperMaxSize); + auto document_key_mapper_or = CreateUriMapper( + *filesystem_, base_dir_, pre_mapping_fbv_, use_persistent_hash_map_); if (!document_key_mapper_or.ok()) { ICING_LOG(ERROR) << document_key_mapper_or.status().error_message() << "Failed to re-init key mapper"; @@ -1771,7 +1832,6 @@ libtextclassifier3::Status DocumentStore::Optimize() { libtextclassifier3::StatusOr<std::vector<DocumentId>> DocumentStore::OptimizeInto(const std::string& new_directory, const LanguageSegmenter* lang_segmenter, - bool namespace_id_fingerprint, OptimizeStatsProto* stats) { // Validates directory if (new_directory == base_dir_) { @@ -1783,7 +1843,8 @@ DocumentStore::OptimizeInto(const std::string& new_directory, auto doc_store_create_result, DocumentStore::Create(filesystem_, new_directory, &clock_, schema_store_, /*force_recovery_and_revalidate_documents=*/false, - namespace_id_fingerprint, compression_level_, + namespace_id_fingerprint_, pre_mapping_fbv_, + use_persistent_hash_map_, compression_level_, /*initialize_stats=*/nullptr)); std::unique_ptr<DocumentStore> new_doc_store = std::move(doc_store_create_result.document_store); diff --git a/icing/store/document-store.h b/icing/store/document-store.h index 3941f6d..92d4286 100644 --- a/icing/store/document-store.h +++ b/icing/store/document-store.h @@ -142,8 +142,8 @@ class DocumentStore { const Filesystem* filesystem, const std::string& base_dir, const Clock* clock, const SchemaStore* schema_store, bool force_recovery_and_revalidate_documents, - bool namespace_id_fingerprint, - int32_t compression_level, + bool namespace_id_fingerprint, bool pre_mapping_fbv, + bool use_persistent_hash_map, int32_t compression_level, InitializeStatsProto* initialize_stats); // Discards all derived data in the document store. @@ -456,7 +456,7 @@ class DocumentStore { // INTERNAL_ERROR on IO error libtextclassifier3::StatusOr<std::vector<DocumentId>> OptimizeInto( const std::string& new_directory, const LanguageSegmenter* lang_segmenter, - bool namespace_id_fingerprint, OptimizeStatsProto* stats = nullptr); + OptimizeStatsProto* stats = nullptr); // Calculates status for a potential Optimize call. Includes how many docs // there are vs how many would be optimized away. And also includes an @@ -488,9 +488,12 @@ class DocumentStore { private: // Use DocumentStore::Create() to instantiate. - DocumentStore(const Filesystem* filesystem, std::string_view base_dir, - const Clock* clock, const SchemaStore* schema_store, - bool namespace_id_fingerprint, int32_t compression_level); + explicit DocumentStore(const Filesystem* filesystem, + std::string_view base_dir, const Clock* clock, + const SchemaStore* schema_store, + bool namespace_id_fingerprint, bool pre_mapping_fbv, + bool use_persistent_hash_map, + int32_t compression_level); const Filesystem* const filesystem_; const std::string base_dir_; @@ -507,6 +510,15 @@ class DocumentStore { // document_key_mapper_ and corpus_mapper_. bool namespace_id_fingerprint_; + // Flag indicating whether memory map max possible file size for underlying + // FileBackedVector before growing the actual file size. + bool pre_mapping_fbv_; + + // Flag indicating whether use persistent hash map as the key mapper (if + // false, then fall back to dynamic trie key mapper). Note: we only use + // persistent hash map for uri mapper if it is true. + bool use_persistent_hash_map_; + const int32_t compression_level_; // A log used to store all documents, it serves as a ground truth of doc diff --git a/icing/store/document-store_benchmark.cc b/icing/store/document-store_benchmark.cc index 75995e9..5b9c568 100644 --- a/icing/store/document-store_benchmark.cc +++ b/icing/store/document-store_benchmark.cc @@ -132,7 +132,8 @@ libtextclassifier3::StatusOr<DocumentStore::CreateResult> CreateDocumentStore( return DocumentStore::Create( filesystem, base_dir, clock, schema_store, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, + /*namespace_id_fingerprint=*/false, /*pre_mapping_fbv=*/false, + /*use_persistent_hash_map=*/false, PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, /*initialize_stats=*/nullptr); } diff --git a/icing/store/document-store_test.cc b/icing/store/document-store_test.cc index 9a1f4a6..a9c47f0 100644 --- a/icing/store/document-store_test.cc +++ b/icing/store/document-store_test.cc @@ -71,6 +71,7 @@ using ::testing::Ge; using ::testing::Gt; using ::testing::HasSubstr; using ::testing::IsEmpty; +using ::testing::IsFalse; using ::testing::IsTrue; using ::testing::Not; using ::testing::Return; @@ -120,7 +121,21 @@ void WriteDocumentLogHeader( sizeof(PortableFileBackedProtoLog<DocumentWrapper>::Header)); } -class DocumentStoreTest : public ::testing::Test { +struct DocumentStoreTestParam { + bool namespace_id_fingerprint; + bool pre_mapping_fbv; + bool use_persistent_hash_map; + + explicit DocumentStoreTestParam(bool namespace_id_fingerprint_in, + bool pre_mapping_fbv_in, + bool use_persistent_hash_map_in) + : namespace_id_fingerprint(namespace_id_fingerprint_in), + pre_mapping_fbv(pre_mapping_fbv_in), + use_persistent_hash_map(use_persistent_hash_map_in) {} +}; + +class DocumentStoreTest + : public ::testing::TestWithParam<DocumentStoreTestParam> { protected: DocumentStoreTest() : test_dir_(GetTestTempDir() + "/icing"), @@ -213,7 +228,7 @@ class DocumentStoreTest : public ::testing::Test { absl_ports::StrCat(document_store_dir_, "/document_store_header"); DocumentStore::Header header; header.magic = DocumentStore::Header::GetCurrentMagic( - /*namespace_id_fingerprint=*/false); + GetParam().namespace_id_fingerprint); header.checksum = 10; // Arbitrary garbage checksum filesystem_.DeleteFile(header_file.c_str()); filesystem_.Write(header_file.c_str(), &header, sizeof(header)); @@ -225,7 +240,8 @@ class DocumentStoreTest : public ::testing::Test { return DocumentStore::Create( filesystem, base_dir, clock, schema_store, /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, + GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, + GetParam().use_persistent_hash_map, PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, /*initialize_stats=*/nullptr); } @@ -254,7 +270,7 @@ class DocumentStoreTest : public ::testing::Test { const int64_t document2_expiration_timestamp_ = 3; // creation + ttl }; -TEST_F(DocumentStoreTest, CreationWithNullPointerShouldFail) { +TEST_P(DocumentStoreTest, CreationWithNullPointerShouldFail) { EXPECT_THAT(CreateDocumentStore(/*filesystem=*/nullptr, document_store_dir_, &fake_clock_, schema_store_.get()), StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); @@ -268,7 +284,7 @@ TEST_F(DocumentStoreTest, CreationWithNullPointerShouldFail) { StatusIs(libtextclassifier3::StatusCode::FAILED_PRECONDITION)); } -TEST_F(DocumentStoreTest, CreationWithBadFilesystemShouldFail) { +TEST_P(DocumentStoreTest, CreationWithBadFilesystemShouldFail) { MockFilesystem mock_filesystem; ON_CALL(mock_filesystem, OpenForWrite(_)).WillByDefault(Return(false)); @@ -277,7 +293,7 @@ TEST_F(DocumentStoreTest, CreationWithBadFilesystemShouldFail) { StatusIs(libtextclassifier3::StatusCode::INTERNAL)); } -TEST_F(DocumentStoreTest, PutAndGetInSameNamespaceOk) { +TEST_P(DocumentStoreTest, PutAndGetInSameNamespaceOk) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -297,7 +313,7 @@ TEST_F(DocumentStoreTest, PutAndGetInSameNamespaceOk) { IsOkAndHolds(EqualsProto(test_document2_))); } -TEST_F(DocumentStoreTest, PutAndGetAcrossNamespacesOk) { +TEST_P(DocumentStoreTest, PutAndGetAcrossNamespacesOk) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -330,7 +346,7 @@ TEST_F(DocumentStoreTest, PutAndGetAcrossNamespacesOk) { // Validates that putting an document with the same key will overwrite previous // document and old doc ids are not getting reused. -TEST_F(DocumentStoreTest, PutSameKey) { +TEST_P(DocumentStoreTest, PutSameKey) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -359,7 +375,7 @@ TEST_F(DocumentStoreTest, PutSameKey) { EXPECT_THAT(doc_store->Put(document3), IsOkAndHolds(Not(document_id1))); } -TEST_F(DocumentStoreTest, IsDocumentExistingWithoutStatus) { +TEST_P(DocumentStoreTest, IsDocumentExistingWithoutStatus) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -395,7 +411,7 @@ TEST_F(DocumentStoreTest, IsDocumentExistingWithoutStatus) { fake_clock_.GetSystemTimeMilliseconds())); } -TEST_F(DocumentStoreTest, GetDeletedDocumentNotFound) { +TEST_P(DocumentStoreTest, GetDeletedDocumentNotFound) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -416,7 +432,7 @@ TEST_F(DocumentStoreTest, GetDeletedDocumentNotFound) { StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } -TEST_F(DocumentStoreTest, GetExpiredDocumentNotFound) { +TEST_P(DocumentStoreTest, GetExpiredDocumentNotFound) { DocumentProto document = DocumentBuilder() .SetKey("namespace", "uri") .SetSchema("email") @@ -451,7 +467,7 @@ TEST_F(DocumentStoreTest, GetExpiredDocumentNotFound) { StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } -TEST_F(DocumentStoreTest, GetInvalidDocumentId) { +TEST_P(DocumentStoreTest, GetInvalidDocumentId) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -478,7 +494,7 @@ TEST_F(DocumentStoreTest, GetInvalidDocumentId) { StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } -TEST_F(DocumentStoreTest, DeleteNonexistentDocumentNotFound) { +TEST_P(DocumentStoreTest, DeleteNonexistentDocumentNotFound) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -504,7 +520,7 @@ TEST_F(DocumentStoreTest, DeleteNonexistentDocumentNotFound) { EXPECT_THAT(document_log_size_before, Eq(document_log_size_after)); } -TEST_F(DocumentStoreTest, DeleteNonexistentDocumentPrintableErrorMessage) { +TEST_P(DocumentStoreTest, DeleteNonexistentDocumentPrintableErrorMessage) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -533,7 +549,7 @@ TEST_F(DocumentStoreTest, DeleteNonexistentDocumentPrintableErrorMessage) { EXPECT_THAT(document_log_size_before, Eq(document_log_size_after)); } -TEST_F(DocumentStoreTest, DeleteAlreadyDeletedDocumentNotFound) { +TEST_P(DocumentStoreTest, DeleteAlreadyDeletedDocumentNotFound) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -555,7 +571,7 @@ TEST_F(DocumentStoreTest, DeleteAlreadyDeletedDocumentNotFound) { StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } -TEST_F(DocumentStoreTest, DeleteByNamespaceOk) { +TEST_P(DocumentStoreTest, DeleteByNamespaceOk) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -599,7 +615,7 @@ TEST_F(DocumentStoreTest, DeleteByNamespaceOk) { StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } -TEST_F(DocumentStoreTest, DeleteByNamespaceNonexistentNamespaceNotFound) { +TEST_P(DocumentStoreTest, DeleteByNamespaceNonexistentNamespaceNotFound) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -624,7 +640,7 @@ TEST_F(DocumentStoreTest, DeleteByNamespaceNonexistentNamespaceNotFound) { EXPECT_THAT(document_log_size_before, Eq(document_log_size_after)); } -TEST_F(DocumentStoreTest, DeleteByNamespaceNoExistingDocumentsNotFound) { +TEST_P(DocumentStoreTest, DeleteByNamespaceNoExistingDocumentsNotFound) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -645,7 +661,7 @@ TEST_F(DocumentStoreTest, DeleteByNamespaceNoExistingDocumentsNotFound) { StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } -TEST_F(DocumentStoreTest, DeleteByNamespaceRecoversOk) { +TEST_P(DocumentStoreTest, DeleteByNamespaceRecoversOk) { DocumentProto document1 = test_document1_; document1.set_namespace_("namespace.1"); document1.set_uri("uri1"); @@ -715,7 +731,7 @@ TEST_F(DocumentStoreTest, DeleteByNamespaceRecoversOk) { StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } -TEST_F(DocumentStoreTest, DeleteBySchemaTypeOk) { +TEST_P(DocumentStoreTest, DeleteBySchemaTypeOk) { SchemaProto schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType("email")) @@ -802,7 +818,7 @@ TEST_F(DocumentStoreTest, DeleteBySchemaTypeOk) { IsOkAndHolds(EqualsProto(person_document))); } -TEST_F(DocumentStoreTest, DeleteBySchemaTypeNonexistentSchemaTypeNotFound) { +TEST_P(DocumentStoreTest, DeleteBySchemaTypeNonexistentSchemaTypeNotFound) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -828,7 +844,7 @@ TEST_F(DocumentStoreTest, DeleteBySchemaTypeNonexistentSchemaTypeNotFound) { EXPECT_THAT(document_log_size_before, Eq(document_log_size_after)); } -TEST_F(DocumentStoreTest, DeleteBySchemaTypeNoExistingDocumentsNotFound) { +TEST_P(DocumentStoreTest, DeleteBySchemaTypeNoExistingDocumentsNotFound) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -846,7 +862,7 @@ TEST_F(DocumentStoreTest, DeleteBySchemaTypeNoExistingDocumentsNotFound) { StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } -TEST_F(DocumentStoreTest, DeleteBySchemaTypeRecoversOk) { +TEST_P(DocumentStoreTest, DeleteBySchemaTypeRecoversOk) { SchemaProto schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType("email")) @@ -926,7 +942,7 @@ TEST_F(DocumentStoreTest, DeleteBySchemaTypeRecoversOk) { IsOkAndHolds(EqualsProto(message_document))); } -TEST_F(DocumentStoreTest, PutDeleteThenPut) { +TEST_P(DocumentStoreTest, PutDeleteThenPut) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -940,7 +956,7 @@ TEST_F(DocumentStoreTest, PutDeleteThenPut) { ICING_EXPECT_OK(doc_store->Put(test_document1_)); } -TEST_F(DocumentStoreTest, DeletedSchemaTypeFromSchemaStoreRecoversOk) { +TEST_P(DocumentStoreTest, DeletedSchemaTypeFromSchemaStoreRecoversOk) { SchemaProto schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder().SetType("email")) @@ -1034,7 +1050,7 @@ TEST_F(DocumentStoreTest, DeletedSchemaTypeFromSchemaStoreRecoversOk) { IsOkAndHolds(EqualsProto(message_document))); } -TEST_F(DocumentStoreTest, OptimizeInto) { +TEST_P(DocumentStoreTest, OptimizeInto) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1078,8 +1094,7 @@ TEST_F(DocumentStoreTest, OptimizeInto) { // Optimizing into the same directory is not allowed EXPECT_THAT( - doc_store->OptimizeInto(document_store_dir_, lang_segmenter_.get(), - /*namespace_id_fingerprint=*/false), + doc_store->OptimizeInto(document_store_dir_, lang_segmenter_.get()), StatusIs(libtextclassifier3::StatusCode::INVALID_ARGUMENT, HasSubstr("directory is the same"))); @@ -1091,8 +1106,7 @@ TEST_F(DocumentStoreTest, OptimizeInto) { // deleted ASSERT_TRUE(filesystem_.DeleteDirectoryRecursively(optimized_dir.c_str())); ASSERT_TRUE(filesystem_.CreateDirectoryRecursively(optimized_dir.c_str())); - EXPECT_THAT(doc_store->OptimizeInto(optimized_dir, lang_segmenter_.get(), - /*namespace_id_fingerprint=*/false), + EXPECT_THAT(doc_store->OptimizeInto(optimized_dir, lang_segmenter_.get()), IsOkAndHolds(ElementsAre(0, 1, 2))); int64_t optimized_size1 = filesystem_.GetFileSize(optimized_document_log.c_str()); @@ -1105,8 +1119,7 @@ TEST_F(DocumentStoreTest, OptimizeInto) { ICING_ASSERT_OK(doc_store->Delete("namespace", "uri1", fake_clock_.GetSystemTimeMilliseconds())); // DocumentId 0 is removed. - EXPECT_THAT(doc_store->OptimizeInto(optimized_dir, lang_segmenter_.get(), - /*namespace_id_fingerprint=*/false), + EXPECT_THAT(doc_store->OptimizeInto(optimized_dir, lang_segmenter_.get()), IsOkAndHolds(ElementsAre(kInvalidDocumentId, 0, 1))); int64_t optimized_size2 = filesystem_.GetFileSize(optimized_document_log.c_str()); @@ -1122,8 +1135,7 @@ TEST_F(DocumentStoreTest, OptimizeInto) { ASSERT_TRUE(filesystem_.CreateDirectoryRecursively(optimized_dir.c_str())); // DocumentId 0 is removed, and DocumentId 2 is expired. EXPECT_THAT( - doc_store->OptimizeInto(optimized_dir, lang_segmenter_.get(), - /*namespace_id_fingerprint=*/false), + doc_store->OptimizeInto(optimized_dir, lang_segmenter_.get()), IsOkAndHolds(ElementsAre(kInvalidDocumentId, 0, kInvalidDocumentId))); int64_t optimized_size3 = filesystem_.GetFileSize(optimized_document_log.c_str()); @@ -1135,8 +1147,7 @@ TEST_F(DocumentStoreTest, OptimizeInto) { ICING_ASSERT_OK(doc_store->Delete("namespace", "uri2", fake_clock_.GetSystemTimeMilliseconds())); // DocumentId 0 and 1 is removed, and DocumentId 2 is expired. - EXPECT_THAT(doc_store->OptimizeInto(optimized_dir, lang_segmenter_.get(), - /*namespace_id_fingerprint=*/false), + EXPECT_THAT(doc_store->OptimizeInto(optimized_dir, lang_segmenter_.get()), IsOkAndHolds(ElementsAre(kInvalidDocumentId, kInvalidDocumentId, kInvalidDocumentId))); int64_t optimized_size4 = @@ -1144,7 +1155,7 @@ TEST_F(DocumentStoreTest, OptimizeInto) { EXPECT_THAT(optimized_size3, Gt(optimized_size4)); } -TEST_F(DocumentStoreTest, OptimizeIntoForEmptyDocumentStore) { +TEST_P(DocumentStoreTest, OptimizeIntoForEmptyDocumentStore) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1154,12 +1165,11 @@ TEST_F(DocumentStoreTest, OptimizeIntoForEmptyDocumentStore) { std::string optimized_dir = document_store_dir_ + "_optimize"; ASSERT_TRUE(filesystem_.DeleteDirectoryRecursively(optimized_dir.c_str())); ASSERT_TRUE(filesystem_.CreateDirectoryRecursively(optimized_dir.c_str())); - EXPECT_THAT(doc_store->OptimizeInto(optimized_dir, lang_segmenter_.get(), - /*namespace_id_fingerprint=*/false), + EXPECT_THAT(doc_store->OptimizeInto(optimized_dir, lang_segmenter_.get()), IsOkAndHolds(IsEmpty())); } -TEST_F(DocumentStoreTest, ShouldRecoverFromDataLoss) { +TEST_P(DocumentStoreTest, ShouldRecoverFromDataLoss) { DocumentId document_id1, document_id2; { // Can put and delete fine. @@ -1250,7 +1260,7 @@ TEST_F(DocumentStoreTest, ShouldRecoverFromDataLoss) { /*num_docs=*/1, /*sum_length_in_tokens=*/4))); } -TEST_F(DocumentStoreTest, ShouldRecoverFromCorruptDerivedFile) { +TEST_P(DocumentStoreTest, ShouldRecoverFromCorruptDerivedFile) { DocumentId document_id1, document_id2; { // Can put and delete fine. @@ -1361,7 +1371,7 @@ TEST_F(DocumentStoreTest, ShouldRecoverFromCorruptDerivedFile) { EXPECT_THAT(actual_scores, Eq(expected_scores)); } -TEST_F(DocumentStoreTest, ShouldRecoverFromDiscardDerivedFiles) { +TEST_P(DocumentStoreTest, ShouldRecoverFromDiscardDerivedFiles) { DocumentId document_id1, document_id2; { // Can put and delete fine. @@ -1459,7 +1469,7 @@ TEST_F(DocumentStoreTest, ShouldRecoverFromDiscardDerivedFiles) { EXPECT_THAT(actual_scores, Eq(expected_scores)); } -TEST_F(DocumentStoreTest, ShouldRecoverFromBadChecksum) { +TEST_P(DocumentStoreTest, ShouldRecoverFromBadChecksum) { DocumentId document_id1, document_id2; { // Can put and delete fine. @@ -1537,7 +1547,7 @@ TEST_F(DocumentStoreTest, ShouldRecoverFromBadChecksum) { /*num_docs=*/1, /*sum_length_in_tokens=*/4))); } -TEST_F(DocumentStoreTest, GetStorageInfo) { +TEST_P(DocumentStoreTest, GetStorageInfo) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1580,7 +1590,7 @@ TEST_F(DocumentStoreTest, GetStorageInfo) { EXPECT_THAT(doc_store_storage_info.document_store_size(), Eq(-1)); } -TEST_F(DocumentStoreTest, MaxDocumentId) { +TEST_P(DocumentStoreTest, MaxDocumentId) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1605,7 +1615,7 @@ TEST_F(DocumentStoreTest, MaxDocumentId) { EXPECT_THAT(doc_store->last_added_document_id(), Eq(document_id2)); } -TEST_F(DocumentStoreTest, GetNamespaceId) { +TEST_P(DocumentStoreTest, GetNamespaceId) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1638,7 +1648,7 @@ TEST_F(DocumentStoreTest, GetNamespaceId) { EXPECT_THAT(doc_store->GetNamespaceId("namespace1"), IsOkAndHolds(Eq(0))); } -TEST_F(DocumentStoreTest, GetDuplicateNamespaceId) { +TEST_P(DocumentStoreTest, GetDuplicateNamespaceId) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1658,7 +1668,7 @@ TEST_F(DocumentStoreTest, GetDuplicateNamespaceId) { EXPECT_THAT(doc_store->GetNamespaceId("namespace"), IsOkAndHolds(Eq(0))); } -TEST_F(DocumentStoreTest, NonexistentNamespaceNotFound) { +TEST_P(DocumentStoreTest, NonexistentNamespaceNotFound) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1670,7 +1680,7 @@ TEST_F(DocumentStoreTest, NonexistentNamespaceNotFound) { StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } -TEST_F(DocumentStoreTest, GetCorpusDuplicateCorpusId) { +TEST_P(DocumentStoreTest, GetCorpusDuplicateCorpusId) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1691,7 +1701,7 @@ TEST_F(DocumentStoreTest, GetCorpusDuplicateCorpusId) { IsOkAndHolds(Eq(0))); } -TEST_F(DocumentStoreTest, GetCorpusId) { +TEST_P(DocumentStoreTest, GetCorpusId) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1726,7 +1736,7 @@ TEST_F(DocumentStoreTest, GetCorpusId) { EXPECT_THAT(doc_store->GetNamespaceId("namespace1"), IsOkAndHolds(Eq(0))); } -TEST_F(DocumentStoreTest, NonexistentCorpusNotFound) { +TEST_P(DocumentStoreTest, NonexistentCorpusNotFound) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1750,7 +1760,7 @@ TEST_F(DocumentStoreTest, NonexistentCorpusNotFound) { StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); } -TEST_F(DocumentStoreTest, GetCorpusAssociatedScoreDataSameCorpus) { +TEST_P(DocumentStoreTest, GetCorpusAssociatedScoreDataSameCorpus) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1775,7 +1785,7 @@ TEST_F(DocumentStoreTest, GetCorpusAssociatedScoreDataSameCorpus) { StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); } -TEST_F(DocumentStoreTest, GetCorpusAssociatedScoreData) { +TEST_P(DocumentStoreTest, GetCorpusAssociatedScoreData) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1813,7 +1823,7 @@ TEST_F(DocumentStoreTest, GetCorpusAssociatedScoreData) { /*num_docs=*/1, /*sum_length_in_tokens=*/5))); } -TEST_F(DocumentStoreTest, NonexistentCorpusAssociatedScoreDataOutOfRange) { +TEST_P(DocumentStoreTest, NonexistentCorpusAssociatedScoreDataOutOfRange) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1825,7 +1835,7 @@ TEST_F(DocumentStoreTest, NonexistentCorpusAssociatedScoreDataOutOfRange) { StatusIs(libtextclassifier3::StatusCode::OUT_OF_RANGE)); } -TEST_F(DocumentStoreTest, GetDocumentAssociatedScoreDataSameCorpus) { +TEST_P(DocumentStoreTest, GetDocumentAssociatedScoreDataSameCorpus) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1869,7 +1879,7 @@ TEST_F(DocumentStoreTest, GetDocumentAssociatedScoreDataSameCorpus) { /*length_in_tokens=*/7))); } -TEST_F(DocumentStoreTest, GetDocumentAssociatedScoreDataDifferentCorpus) { +TEST_P(DocumentStoreTest, GetDocumentAssociatedScoreDataDifferentCorpus) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1913,7 +1923,7 @@ TEST_F(DocumentStoreTest, GetDocumentAssociatedScoreDataDifferentCorpus) { /*length_in_tokens=*/7))); } -TEST_F(DocumentStoreTest, NonexistentDocumentAssociatedScoreDataNotFound) { +TEST_P(DocumentStoreTest, NonexistentDocumentAssociatedScoreDataNotFound) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1925,7 +1935,7 @@ TEST_F(DocumentStoreTest, NonexistentDocumentAssociatedScoreDataNotFound) { StatusIs(libtextclassifier3::StatusCode::NOT_FOUND)); } -TEST_F(DocumentStoreTest, NonexistentDocumentFilterDataNotFound) { +TEST_P(DocumentStoreTest, NonexistentDocumentFilterDataNotFound) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1937,7 +1947,7 @@ TEST_F(DocumentStoreTest, NonexistentDocumentFilterDataNotFound) { /*document_id=*/0, fake_clock_.GetSystemTimeMilliseconds())); } -TEST_F(DocumentStoreTest, DeleteClearsFilterCache) { +TEST_P(DocumentStoreTest, DeleteClearsFilterCache) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1964,7 +1974,7 @@ TEST_F(DocumentStoreTest, DeleteClearsFilterCache) { document_id, fake_clock_.GetSystemTimeMilliseconds())); } -TEST_F(DocumentStoreTest, DeleteClearsScoreCache) { +TEST_P(DocumentStoreTest, DeleteClearsScoreCache) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -1993,7 +2003,7 @@ TEST_F(DocumentStoreTest, DeleteClearsScoreCache) { /*length_in_tokens=*/0))); } -TEST_F(DocumentStoreTest, DeleteShouldPreventUsageScores) { +TEST_P(DocumentStoreTest, DeleteShouldPreventUsageScores) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -2032,7 +2042,7 @@ TEST_F(DocumentStoreTest, DeleteShouldPreventUsageScores) { document_id, fake_clock_.GetSystemTimeMilliseconds())); } -TEST_F(DocumentStoreTest, ExpirationShouldPreventUsageScores) { +TEST_P(DocumentStoreTest, ExpirationShouldPreventUsageScores) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -2082,7 +2092,7 @@ TEST_F(DocumentStoreTest, ExpirationShouldPreventUsageScores) { document_id, fake_clock_.GetSystemTimeMilliseconds())); } -TEST_F(DocumentStoreTest, +TEST_P(DocumentStoreTest, ExpirationTimestampIsSumOfNonZeroTtlAndCreationTimestamp) { DocumentProto document = DocumentBuilder() .SetKey("namespace1", "1") @@ -2109,7 +2119,7 @@ TEST_F(DocumentStoreTest, /*expiration_timestamp_ms=*/1100))); } -TEST_F(DocumentStoreTest, ExpirationTimestampIsInt64MaxIfTtlIsZero) { +TEST_P(DocumentStoreTest, ExpirationTimestampIsInt64MaxIfTtlIsZero) { DocumentProto document = DocumentBuilder() .SetKey("namespace1", "1") .SetSchema("email") @@ -2139,7 +2149,7 @@ TEST_F(DocumentStoreTest, ExpirationTimestampIsInt64MaxIfTtlIsZero) { /*expiration_timestamp_ms=*/std::numeric_limits<int64_t>::max()))); } -TEST_F(DocumentStoreTest, ExpirationTimestampIsInt64MaxOnOverflow) { +TEST_P(DocumentStoreTest, ExpirationTimestampIsInt64MaxOnOverflow) { DocumentProto document = DocumentBuilder() .SetKey("namespace1", "1") @@ -2170,7 +2180,7 @@ TEST_F(DocumentStoreTest, ExpirationTimestampIsInt64MaxOnOverflow) { /*expiration_timestamp_ms=*/std::numeric_limits<int64_t>::max()))); } -TEST_F(DocumentStoreTest, CreationTimestampShouldBePopulated) { +TEST_P(DocumentStoreTest, CreationTimestampShouldBePopulated) { // Creates a document without a given creation timestamp DocumentProto document_without_creation_timestamp = DocumentBuilder() @@ -2201,7 +2211,7 @@ TEST_F(DocumentStoreTest, CreationTimestampShouldBePopulated) { Eq(fake_real_time)); } -TEST_F(DocumentStoreTest, ShouldWriteAndReadScoresCorrectly) { +TEST_P(DocumentStoreTest, ShouldWriteAndReadScoresCorrectly) { DocumentProto document1 = DocumentBuilder() .SetKey("icing", "email/1") .SetSchema("email") @@ -2240,7 +2250,7 @@ TEST_F(DocumentStoreTest, ShouldWriteAndReadScoresCorrectly) { /*length_in_tokens=*/0))); } -TEST_F(DocumentStoreTest, ComputeChecksumSameBetweenCalls) { +TEST_P(DocumentStoreTest, ComputeChecksumSameBetweenCalls) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -2255,7 +2265,7 @@ TEST_F(DocumentStoreTest, ComputeChecksumSameBetweenCalls) { EXPECT_THAT(document_store->ComputeChecksum(), IsOkAndHolds(checksum)); } -TEST_F(DocumentStoreTest, ComputeChecksumSameAcrossInstances) { +TEST_P(DocumentStoreTest, ComputeChecksumSameAcrossInstances) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -2276,7 +2286,7 @@ TEST_F(DocumentStoreTest, ComputeChecksumSameAcrossInstances) { EXPECT_THAT(document_store->ComputeChecksum(), IsOkAndHolds(checksum)); } -TEST_F(DocumentStoreTest, ComputeChecksumChangesOnNewDocument) { +TEST_P(DocumentStoreTest, ComputeChecksumChangesOnNewDocument) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -2292,7 +2302,7 @@ TEST_F(DocumentStoreTest, ComputeChecksumChangesOnNewDocument) { IsOkAndHolds(Not(Eq(checksum)))); } -TEST_F(DocumentStoreTest, ComputeChecksumDoesntChangeOnNewUsage) { +TEST_P(DocumentStoreTest, ComputeChecksumDoesntChangeOnNewUsage) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -2310,7 +2320,7 @@ TEST_F(DocumentStoreTest, ComputeChecksumDoesntChangeOnNewUsage) { EXPECT_THAT(document_store->ComputeChecksum(), IsOkAndHolds(Eq(checksum))); } -TEST_F(DocumentStoreTest, RegenerateDerivedFilesSkipsUnknownSchemaTypeIds) { +TEST_P(DocumentStoreTest, RegenerateDerivedFilesSkipsUnknownSchemaTypeIds) { const std::string schema_store_dir = schema_store_dir_ + "_custom"; DocumentId email_document_id; @@ -2445,7 +2455,7 @@ TEST_F(DocumentStoreTest, RegenerateDerivedFilesSkipsUnknownSchemaTypeIds) { Eq(message_expiration_timestamp)); } -TEST_F(DocumentStoreTest, UpdateSchemaStoreUpdatesSchemaTypeIds) { +TEST_P(DocumentStoreTest, UpdateSchemaStoreUpdatesSchemaTypeIds) { const std::string schema_store_dir = test_dir_ + "_custom"; filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); @@ -2541,7 +2551,7 @@ TEST_F(DocumentStoreTest, UpdateSchemaStoreUpdatesSchemaTypeIds) { EXPECT_THAT(message_data.schema_type_id(), Eq(new_message_schema_type_id)); } -TEST_F(DocumentStoreTest, UpdateSchemaStoreDeletesInvalidDocuments) { +TEST_P(DocumentStoreTest, UpdateSchemaStoreDeletesInvalidDocuments) { const std::string schema_store_dir = test_dir_ + "_custom"; filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); @@ -2617,7 +2627,7 @@ TEST_F(DocumentStoreTest, UpdateSchemaStoreDeletesInvalidDocuments) { IsOkAndHolds(EqualsProto(email_with_subject))); } -TEST_F(DocumentStoreTest, +TEST_P(DocumentStoreTest, UpdateSchemaStoreDeletesDocumentsByDeletedSchemaType) { const std::string schema_store_dir = test_dir_ + "_custom"; filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); @@ -2691,7 +2701,7 @@ TEST_F(DocumentStoreTest, IsOkAndHolds(EqualsProto(message_document))); } -TEST_F(DocumentStoreTest, OptimizedUpdateSchemaStoreUpdatesSchemaTypeIds) { +TEST_P(DocumentStoreTest, OptimizedUpdateSchemaStoreUpdatesSchemaTypeIds) { const std::string schema_store_dir = test_dir_ + "_custom"; filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); @@ -2790,7 +2800,7 @@ TEST_F(DocumentStoreTest, OptimizedUpdateSchemaStoreUpdatesSchemaTypeIds) { EXPECT_THAT(message_data.schema_type_id(), Eq(new_message_schema_type_id)); } -TEST_F(DocumentStoreTest, OptimizedUpdateSchemaStoreDeletesInvalidDocuments) { +TEST_P(DocumentStoreTest, OptimizedUpdateSchemaStoreDeletesInvalidDocuments) { const std::string schema_store_dir = test_dir_ + "_custom"; filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); @@ -2869,7 +2879,7 @@ TEST_F(DocumentStoreTest, OptimizedUpdateSchemaStoreDeletesInvalidDocuments) { IsOkAndHolds(EqualsProto(email_with_subject))); } -TEST_F(DocumentStoreTest, +TEST_P(DocumentStoreTest, OptimizedUpdateSchemaStoreDeletesDocumentsByDeletedSchemaType) { const std::string schema_store_dir = test_dir_ + "_custom"; filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); @@ -2945,7 +2955,7 @@ TEST_F(DocumentStoreTest, IsOkAndHolds(EqualsProto(message_document))); } -TEST_F(DocumentStoreTest, GetOptimizeInfo) { +TEST_P(DocumentStoreTest, GetOptimizeInfo) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -2983,8 +2993,7 @@ TEST_F(DocumentStoreTest, GetOptimizeInfo) { EXPECT_TRUE(filesystem_.DeleteDirectoryRecursively(optimized_dir.c_str())); EXPECT_TRUE(filesystem_.CreateDirectoryRecursively(optimized_dir.c_str())); ICING_ASSERT_OK( - document_store->OptimizeInto(optimized_dir, lang_segmenter_.get(), - /*namespace_id_fingerprint=*/false)); + document_store->OptimizeInto(optimized_dir, lang_segmenter_.get())); document_store.reset(); ICING_ASSERT_OK_AND_ASSIGN( create_result, CreateDocumentStore(&filesystem_, optimized_dir, @@ -2999,7 +3008,7 @@ TEST_F(DocumentStoreTest, GetOptimizeInfo) { EXPECT_THAT(optimize_info.estimated_optimizable_bytes, Eq(0)); } -TEST_F(DocumentStoreTest, GetAllNamespaces) { +TEST_P(DocumentStoreTest, GetAllNamespaces) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -3071,7 +3080,7 @@ TEST_F(DocumentStoreTest, GetAllNamespaces) { UnorderedElementsAre("namespace1")); } -TEST_F(DocumentStoreTest, ReportUsageWithDifferentTimestampsAndGetUsageScores) { +TEST_P(DocumentStoreTest, ReportUsageWithDifferentTimestampsAndGetUsageScores) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -3163,7 +3172,7 @@ TEST_F(DocumentStoreTest, ReportUsageWithDifferentTimestampsAndGetUsageScores) { EXPECT_THAT(actual_scores, Eq(expected_scores)); } -TEST_F(DocumentStoreTest, ReportUsageWithDifferentTypesAndGetUsageScores) { +TEST_P(DocumentStoreTest, ReportUsageWithDifferentTypesAndGetUsageScores) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -3213,7 +3222,7 @@ TEST_F(DocumentStoreTest, ReportUsageWithDifferentTypesAndGetUsageScores) { EXPECT_THAT(actual_scores, Eq(expected_scores)); } -TEST_F(DocumentStoreTest, UsageScoresShouldNotBeClearedOnChecksumMismatch) { +TEST_P(DocumentStoreTest, UsageScoresShouldNotBeClearedOnChecksumMismatch) { UsageStore::UsageScores expected_scores; DocumentId document_id; { @@ -3258,7 +3267,7 @@ TEST_F(DocumentStoreTest, UsageScoresShouldNotBeClearedOnChecksumMismatch) { EXPECT_THAT(actual_scores, Eq(expected_scores)); } -TEST_F(DocumentStoreTest, UsageScoresShouldBeAvailableAfterDataLoss) { +TEST_P(DocumentStoreTest, UsageScoresShouldBeAvailableAfterDataLoss) { UsageStore::UsageScores expected_scores; DocumentId document_id; { @@ -3314,7 +3323,7 @@ TEST_F(DocumentStoreTest, UsageScoresShouldBeAvailableAfterDataLoss) { EXPECT_THAT(actual_scores, Eq(expected_scores)); } -TEST_F(DocumentStoreTest, UsageScoresShouldBeCopiedOverToUpdatedDocument) { +TEST_P(DocumentStoreTest, UsageScoresShouldBeCopiedOverToUpdatedDocument) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -3355,7 +3364,7 @@ TEST_F(DocumentStoreTest, UsageScoresShouldBeCopiedOverToUpdatedDocument) { EXPECT_THAT(actual_scores, Eq(expected_scores)); } -TEST_F(DocumentStoreTest, UsageScoresShouldPersistOnOptimize) { +TEST_P(DocumentStoreTest, UsageScoresShouldPersistOnOptimize) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -3390,8 +3399,7 @@ TEST_F(DocumentStoreTest, UsageScoresShouldPersistOnOptimize) { std::string optimized_dir = document_store_dir_ + "/optimize_test"; filesystem_.CreateDirectoryRecursively(optimized_dir.c_str()); ICING_ASSERT_OK( - document_store->OptimizeInto(optimized_dir, lang_segmenter_.get(), - /*namespace_id_fingerprint=*/false)); + document_store->OptimizeInto(optimized_dir, lang_segmenter_.get())); // Get optimized document store ICING_ASSERT_OK_AND_ASSIGN( @@ -3409,7 +3417,7 @@ TEST_F(DocumentStoreTest, UsageScoresShouldPersistOnOptimize) { EXPECT_THAT(actual_scores, Eq(expected_scores)); } -TEST_F(DocumentStoreTest, DetectPartialDataLoss) { +TEST_P(DocumentStoreTest, DetectPartialDataLoss) { { // Can put and delete fine. ICING_ASSERT_OK_AND_ASSIGN( @@ -3450,7 +3458,7 @@ TEST_F(DocumentStoreTest, DetectPartialDataLoss) { ASSERT_THAT(create_result.data_loss, Eq(DataLoss::PARTIAL)); } -TEST_F(DocumentStoreTest, DetectCompleteDataLoss) { +TEST_P(DocumentStoreTest, DetectCompleteDataLoss) { int64_t corruptible_offset; const std::string document_log_file = absl_ports::StrCat( document_store_dir_, "/", DocumentLogCreator::GetDocumentLogFilename()); @@ -3515,7 +3523,7 @@ TEST_F(DocumentStoreTest, DetectCompleteDataLoss) { ASSERT_THAT(create_result.data_loss, Eq(DataLoss::COMPLETE)); } -TEST_F(DocumentStoreTest, LoadScoreCacheAndInitializeSuccessfully) { +TEST_P(DocumentStoreTest, LoadScoreCacheAndInitializeSuccessfully) { // The directory testdata/score_cache_without_length_in_tokens/document_store // contains only the scoring_cache and the document_store_header (holding the // crc for the scoring_cache). If the current code is compatible with the @@ -3557,7 +3565,8 @@ TEST_F(DocumentStoreTest, LoadScoreCacheAndInitializeSuccessfully) { DocumentStore::Create( &filesystem_, document_store_dir_, &fake_clock_, schema_store_.get(), /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, + GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, + GetParam().use_persistent_hash_map, PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, &initialize_stats)); std::unique_ptr<DocumentStore> doc_store = @@ -3568,7 +3577,7 @@ TEST_F(DocumentStoreTest, LoadScoreCacheAndInitializeSuccessfully) { InitializeStatsProto::LEGACY_DOCUMENT_LOG_FORMAT); } -TEST_F(DocumentStoreTest, DocumentStoreStorageInfo) { +TEST_P(DocumentStoreTest, DocumentStoreStorageInfo) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -3678,7 +3687,7 @@ TEST_F(DocumentStoreTest, DocumentStoreStorageInfo) { Eq(0)); } -TEST_F(DocumentStoreTest, InitializeForceRecoveryUpdatesTypeIds) { +TEST_P(DocumentStoreTest, InitializeForceRecoveryUpdatesTypeIds) { // Start fresh and set the schema with one type. filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); @@ -3768,13 +3777,14 @@ TEST_F(DocumentStoreTest, InitializeForceRecoveryUpdatesTypeIds) { InitializeStatsProto initialize_stats; ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, - schema_store.get(), - /*force_recovery_and_revalidate_documents=*/true, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - &initialize_stats)); + DocumentStore::Create( + &filesystem_, document_store_dir_, &fake_clock_, schema_store.get(), + /*force_recovery_and_revalidate_documents=*/true, + GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, + GetParam().use_persistent_hash_map, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + &initialize_stats)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -3789,7 +3799,7 @@ TEST_F(DocumentStoreTest, InitializeForceRecoveryUpdatesTypeIds) { } } -TEST_F(DocumentStoreTest, InitializeDontForceRecoveryDoesntUpdateTypeIds) { +TEST_P(DocumentStoreTest, InitializeDontForceRecoveryDoesntUpdateTypeIds) { // Start fresh and set the schema with one type. filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); @@ -3892,7 +3902,7 @@ TEST_F(DocumentStoreTest, InitializeDontForceRecoveryDoesntUpdateTypeIds) { } } -TEST_F(DocumentStoreTest, InitializeForceRecoveryDeletesInvalidDocument) { +TEST_P(DocumentStoreTest, InitializeForceRecoveryDeletesInvalidDocument) { // Start fresh and set the schema with one type. filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); @@ -3987,13 +3997,14 @@ TEST_F(DocumentStoreTest, InitializeForceRecoveryDeletesInvalidDocument) { CorruptDocStoreHeaderChecksumFile(); ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, - DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, - schema_store.get(), - /*force_recovery_and_revalidate_documents=*/true, - /*namespace_id_fingerprint=*/false, - PortableFileBackedProtoLog< - DocumentWrapper>::kDeflateCompressionLevel, - /*initialize_stats=*/nullptr)); + DocumentStore::Create( + &filesystem_, document_store_dir_, &fake_clock_, schema_store.get(), + /*force_recovery_and_revalidate_documents=*/true, + GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, + GetParam().use_persistent_hash_map, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); std::unique_ptr<DocumentStore> doc_store = std::move(create_result.document_store); @@ -4005,7 +4016,7 @@ TEST_F(DocumentStoreTest, InitializeForceRecoveryDeletesInvalidDocument) { } } -TEST_F(DocumentStoreTest, InitializeDontForceRecoveryKeepsInvalidDocument) { +TEST_P(DocumentStoreTest, InitializeDontForceRecoveryKeepsInvalidDocument) { // Start fresh and set the schema with one type. filesystem_.DeleteDirectoryRecursively(test_dir_.c_str()); filesystem_.CreateDirectoryRecursively(test_dir_.c_str()); @@ -4114,7 +4125,7 @@ TEST_F(DocumentStoreTest, InitializeDontForceRecoveryKeepsInvalidDocument) { } } -TEST_F(DocumentStoreTest, MigrateToPortableFileBackedProtoLog) { +TEST_P(DocumentStoreTest, MigrateToPortableFileBackedProtoLog) { // Set up schema. SchemaProto schema = SchemaBuilder() @@ -4182,7 +4193,8 @@ TEST_F(DocumentStoreTest, MigrateToPortableFileBackedProtoLog) { DocumentStore::Create( &filesystem_, document_store_dir, &fake_clock_, schema_store.get(), /*force_recovery_and_revalidate_documents=*/false, - /*namespace_id_fingerprint=*/false, + GetParam().pre_mapping_fbv, GetParam().use_persistent_hash_map, + GetParam().namespace_id_fingerprint, PortableFileBackedProtoLog<DocumentWrapper>::kDeflateCompressionLevel, &initialize_stats)); std::unique_ptr<DocumentStore> document_store = @@ -4240,7 +4252,7 @@ TEST_F(DocumentStoreTest, MigrateToPortableFileBackedProtoLog) { IsOkAndHolds(EqualsProto(document3))); } -TEST_F(DocumentStoreTest, GetDebugInfo) { +TEST_P(DocumentStoreTest, GetDebugInfo) { SchemaProto schema = SchemaBuilder() .AddType(SchemaTypeConfigBuilder() @@ -4365,7 +4377,7 @@ TEST_F(DocumentStoreTest, GetDebugInfo) { EXPECT_THAT(out3.corpus_info(), IsEmpty()); } -TEST_F(DocumentStoreTest, GetDebugInfoWithoutSchema) { +TEST_P(DocumentStoreTest, GetDebugInfoWithoutSchema) { std::string schema_store_dir = schema_store_dir_ + "_custom"; filesystem_.DeleteDirectoryRecursively(schema_store_dir.c_str()); filesystem_.CreateDirectoryRecursively(schema_store_dir.c_str()); @@ -4389,7 +4401,7 @@ TEST_F(DocumentStoreTest, GetDebugInfoWithoutSchema) { EXPECT_THAT(out.corpus_info(), IsEmpty()); } -TEST_F(DocumentStoreTest, GetDebugInfoForEmptyDocumentStore) { +TEST_P(DocumentStoreTest, GetDebugInfoForEmptyDocumentStore) { ICING_ASSERT_OK_AND_ASSIGN( DocumentStore::CreateResult create_result, CreateDocumentStore(&filesystem_, document_store_dir_, &fake_clock_, @@ -4406,6 +4418,198 @@ TEST_F(DocumentStoreTest, GetDebugInfoForEmptyDocumentStore) { EXPECT_THAT(out.corpus_info(), IsEmpty()); } +TEST_P(DocumentStoreTest, SwitchKeyMapperTypeShouldRegenerateDerivedFiles) { + std::string dynamic_trie_uri_mapper_dir = + document_store_dir_ + "/key_mapper_dir"; + std::string persistent_hash_map_uri_mapper_dir = + document_store_dir_ + "/uri_mapper"; + DocumentId document_id1; + { + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, + schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + GetParam().namespace_id_fingerprint, + GetParam().pre_mapping_fbv, + GetParam().use_persistent_hash_map, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); + + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + ICING_ASSERT_OK_AND_ASSIGN(document_id1, doc_store->Put(test_document1_)); + + if (GetParam().use_persistent_hash_map) { + EXPECT_THAT(filesystem_.DirectoryExists( + persistent_hash_map_uri_mapper_dir.c_str()), + IsTrue()); + EXPECT_THAT( + filesystem_.DirectoryExists(dynamic_trie_uri_mapper_dir.c_str()), + IsFalse()); + } else { + EXPECT_THAT(filesystem_.DirectoryExists( + persistent_hash_map_uri_mapper_dir.c_str()), + IsFalse()); + EXPECT_THAT( + filesystem_.DirectoryExists(dynamic_trie_uri_mapper_dir.c_str()), + IsTrue()); + } + } + + // Switch key mapper. We should get I/O error and derived files should be + // regenerated. + { + bool switch_key_mapper_flag = !GetParam().use_persistent_hash_map; + InitializeStatsProto initialize_stats; + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + DocumentStore::Create( + &filesystem_, document_store_dir_, &fake_clock_, + schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + GetParam().namespace_id_fingerprint, GetParam().pre_mapping_fbv, + /*use_persistent_hash_map=*/switch_key_mapper_flag, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + &initialize_stats)); + EXPECT_THAT(initialize_stats.document_store_recovery_cause(), + Eq(InitializeStatsProto::IO_ERROR)); + + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + EXPECT_THAT(doc_store->GetDocumentId(test_document1_.namespace_(), + test_document1_.uri()), + IsOkAndHolds(document_id1)); + + if (switch_key_mapper_flag) { + EXPECT_THAT(filesystem_.DirectoryExists( + persistent_hash_map_uri_mapper_dir.c_str()), + IsTrue()); + EXPECT_THAT( + filesystem_.DirectoryExists(dynamic_trie_uri_mapper_dir.c_str()), + IsFalse()); + } else { + EXPECT_THAT(filesystem_.DirectoryExists( + persistent_hash_map_uri_mapper_dir.c_str()), + IsFalse()); + EXPECT_THAT( + filesystem_.DirectoryExists(dynamic_trie_uri_mapper_dir.c_str()), + IsTrue()); + } + } +} + +TEST_P(DocumentStoreTest, SameKeyMapperTypeShouldNotRegenerateDerivedFiles) { + std::string dynamic_trie_uri_mapper_dir = + document_store_dir_ + "/key_mapper_dir"; + std::string persistent_hash_map_uri_mapper_dir = + document_store_dir_ + "/uri_mapper"; + DocumentId document_id1; + { + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, + schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + GetParam().namespace_id_fingerprint, + GetParam().pre_mapping_fbv, + GetParam().use_persistent_hash_map, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + /*initialize_stats=*/nullptr)); + + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + ICING_ASSERT_OK_AND_ASSIGN(document_id1, doc_store->Put(test_document1_)); + + if (GetParam().use_persistent_hash_map) { + EXPECT_THAT(filesystem_.DirectoryExists( + persistent_hash_map_uri_mapper_dir.c_str()), + IsTrue()); + EXPECT_THAT( + filesystem_.DirectoryExists(dynamic_trie_uri_mapper_dir.c_str()), + IsFalse()); + } else { + EXPECT_THAT(filesystem_.DirectoryExists( + persistent_hash_map_uri_mapper_dir.c_str()), + IsFalse()); + EXPECT_THAT( + filesystem_.DirectoryExists(dynamic_trie_uri_mapper_dir.c_str()), + IsTrue()); + } + } + + // Use the same key mapper type. Derived files should not be regenerated. + { + InitializeStatsProto initialize_stats; + ICING_ASSERT_OK_AND_ASSIGN( + DocumentStore::CreateResult create_result, + DocumentStore::Create(&filesystem_, document_store_dir_, &fake_clock_, + schema_store_.get(), + /*force_recovery_and_revalidate_documents=*/false, + GetParam().namespace_id_fingerprint, + GetParam().pre_mapping_fbv, + GetParam().use_persistent_hash_map, + PortableFileBackedProtoLog< + DocumentWrapper>::kDeflateCompressionLevel, + &initialize_stats)); + EXPECT_THAT(initialize_stats.document_store_recovery_cause(), + Eq(InitializeStatsProto::NONE)); + + std::unique_ptr<DocumentStore> doc_store = + std::move(create_result.document_store); + EXPECT_THAT(doc_store->GetDocumentId(test_document1_.namespace_(), + test_document1_.uri()), + IsOkAndHolds(document_id1)); + + if (GetParam().use_persistent_hash_map) { + EXPECT_THAT(filesystem_.DirectoryExists( + persistent_hash_map_uri_mapper_dir.c_str()), + IsTrue()); + EXPECT_THAT( + filesystem_.DirectoryExists(dynamic_trie_uri_mapper_dir.c_str()), + IsFalse()); + } else { + EXPECT_THAT(filesystem_.DirectoryExists( + persistent_hash_map_uri_mapper_dir.c_str()), + IsFalse()); + EXPECT_THAT( + filesystem_.DirectoryExists(dynamic_trie_uri_mapper_dir.c_str()), + IsTrue()); + } + } +} + +INSTANTIATE_TEST_SUITE_P( + DocumentStoreTest, DocumentStoreTest, + testing::Values( + DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/false, + /*pre_mapping_fbv_in=*/false, + /*use_persistent_hash_map_in=*/false), + DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/true, + /*pre_mapping_fbv_in=*/false, + /*use_persistent_hash_map_in=*/false), + DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/false, + /*pre_mapping_fbv_in=*/true, + /*use_persistent_hash_map_in=*/false), + DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/true, + /*pre_mapping_fbv_in=*/true, + /*use_persistent_hash_map_in=*/false), + DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/false, + /*pre_mapping_fbv_in=*/false, + /*use_persistent_hash_map_in=*/true), + DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/true, + /*pre_mapping_fbv_in=*/false, + /*use_persistent_hash_map_in=*/true), + DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/false, + /*pre_mapping_fbv_in=*/true, + /*use_persistent_hash_map_in=*/true), + DocumentStoreTestParam(/*namespace_id_fingerprint_in=*/true, + /*pre_mapping_fbv_in=*/true, + /*use_persistent_hash_map_in=*/true))); + } // namespace } // namespace lib diff --git a/icing/testing/common-matchers.h b/icing/testing/common-matchers.h index bbc1a59..c6500db 100644 --- a/icing/testing/common-matchers.h +++ b/icing/testing/common-matchers.h @@ -241,7 +241,9 @@ MATCHER_P(EqualsSetSchemaResult, expected, "") { actual.schema_types_changed_fully_compatible_by_name == expected.schema_types_changed_fully_compatible_by_name && actual.schema_types_index_incompatible_by_name == - expected.schema_types_index_incompatible_by_name) { + expected.schema_types_index_incompatible_by_name && + actual.schema_types_join_incompatible_by_name == + expected.schema_types_join_incompatible_by_name) { return true; } @@ -338,6 +340,21 @@ MATCHER_P(EqualsSetSchemaResult, expected, "") { ","), "]"); + // Format schema_types_join_incompatible_by_name + std::string actual_schema_types_join_incompatible_by_name = + absl_ports::StrCat( + "[", + absl_ports::StrJoin(actual.schema_types_join_incompatible_by_name, + ","), + "]"); + + std::string expected_schema_types_join_incompatible_by_name = + absl_ports::StrCat( + "[", + absl_ports::StrJoin(expected.schema_types_join_incompatible_by_name, + ","), + "]"); + *result_listener << IcingStringUtil::StringPrintf( "\nExpected {\n" "\tsuccess=%d,\n" @@ -347,8 +364,9 @@ MATCHER_P(EqualsSetSchemaResult, expected, "") { "\tschema_types_incompatible_by_name=%s,\n" "\tschema_types_incompatible_by_id=%s\n" "\tschema_types_new_by_name=%s,\n" - "\tschema_types_index_incompatible_by_name=%s,\n" "\tschema_types_changed_fully_compatible_by_name=%s\n" + "\tschema_types_index_incompatible_by_name=%s,\n" + "\tschema_types_join_incompatible_by_name=%s\n" "}\n" "Actual {\n" "\tsuccess=%d,\n" @@ -358,8 +376,9 @@ MATCHER_P(EqualsSetSchemaResult, expected, "") { "\tschema_types_incompatible_by_name=%s,\n" "\tschema_types_incompatible_by_id=%s\n" "\tschema_types_new_by_name=%s,\n" - "\tschema_types_index_incompatible_by_name=%s,\n" "\tschema_types_changed_fully_compatible_by_name=%s\n" + "\tschema_types_index_incompatible_by_name=%s,\n" + "\tschema_types_join_incompatible_by_name=%s\n" "}\n", expected.success, expected_old_schema_type_ids_changed.c_str(), expected_schema_types_deleted_by_name.c_str(), @@ -368,7 +387,8 @@ MATCHER_P(EqualsSetSchemaResult, expected, "") { expected_schema_types_incompatible_by_id.c_str(), expected_schema_types_new_by_name.c_str(), expected_schema_types_changed_fully_compatible_by_name.c_str(), - expected_schema_types_index_incompatible_by_name.c_str(), actual.success, + expected_schema_types_index_incompatible_by_name.c_str(), + expected_schema_types_join_incompatible_by_name.c_str(), actual.success, actual_old_schema_type_ids_changed.c_str(), actual_schema_types_deleted_by_name.c_str(), actual_schema_types_deleted_by_id.c_str(), @@ -376,7 +396,8 @@ MATCHER_P(EqualsSetSchemaResult, expected, "") { actual_schema_types_incompatible_by_id.c_str(), actual_schema_types_new_by_name.c_str(), actual_schema_types_changed_fully_compatible_by_name.c_str(), - actual_schema_types_index_incompatible_by_name.c_str()); + actual_schema_types_index_incompatible_by_name.c_str(), + actual_schema_types_join_incompatible_by_name.c_str()); return false; } diff --git a/java/src/com/google/android/icing/IcingSearchEngine.java b/java/src/com/google/android/icing/IcingSearchEngine.java index 47b94a5..79fcdb8 100644 --- a/java/src/com/google/android/icing/IcingSearchEngine.java +++ b/java/src/com/google/android/icing/IcingSearchEngine.java @@ -77,6 +77,7 @@ public class IcingSearchEngine implements IcingSearchEngineInterface { icingSearchEngineImpl.close(); } + @SuppressWarnings("deprecation") @Override protected void finalize() throws Throwable { icingSearchEngineImpl.close(); diff --git a/java/src/com/google/android/icing/IcingSearchEngineImpl.java b/java/src/com/google/android/icing/IcingSearchEngineImpl.java index 8e79a88..57744c4 100644 --- a/java/src/com/google/android/icing/IcingSearchEngineImpl.java +++ b/java/src/com/google/android/icing/IcingSearchEngineImpl.java @@ -71,6 +71,7 @@ public class IcingSearchEngineImpl implements Closeable { closed = true; } + @SuppressWarnings("deprecation") @Override protected void finalize() throws Throwable { close(); diff --git a/java/tests/instrumentation/src/com/google/android/icing/IcingSearchEngineTest.java b/java/tests/instrumentation/src/com/google/android/icing/IcingSearchEngineTest.java index 1ed2d9a..2bbd621 100644 --- a/java/tests/instrumentation/src/com/google/android/icing/IcingSearchEngineTest.java +++ b/java/tests/instrumentation/src/com/google/android/icing/IcingSearchEngineTest.java @@ -139,26 +139,6 @@ public final class IcingSearchEngineTest { } @Test - public void testSetAndGetSchema() throws Exception { - assertStatusOk(icingSearchEngine.initialize().getStatus()); - - SchemaTypeConfigProto emailTypeConfig = createEmailTypeConfig(); - SchemaProto schema = SchemaProto.newBuilder().addTypes(emailTypeConfig).build(); - SetSchemaResultProto setSchemaResultProto = - icingSearchEngine.setSchema(schema, /*ignoreErrorsAndDeleteDocuments=*/ false); - assertStatusOk(setSchemaResultProto.getStatus()); - - GetSchemaResultProto getSchemaResultProto = icingSearchEngine.getSchema(); - assertStatusOk(getSchemaResultProto.getStatus()); - assertThat(getSchemaResultProto.getSchema()).isEqualTo(schema); - - GetSchemaTypeResultProto getSchemaTypeResultProto = - icingSearchEngine.getSchemaType(emailTypeConfig.getSchemaType()); - assertStatusOk(getSchemaTypeResultProto.getStatus()); - assertThat(getSchemaTypeResultProto.getSchemaTypeConfig()).isEqualTo(emailTypeConfig); - } - - @Test public void testPutAndGetDocuments() throws Exception { assertStatusOk(icingSearchEngine.initialize().getStatus()); diff --git a/proto/icing/proto/initialize.proto b/proto/icing/proto/initialize.proto index d4b1aee..958767b 100644 --- a/proto/icing/proto/initialize.proto +++ b/proto/icing/proto/initialize.proto @@ -23,7 +23,7 @@ option java_package = "com.google.android.icing.proto"; option java_multiple_files = true; option objc_class_prefix = "ICNG"; -// Next tag: 11 +// Next tag: 14 message IcingSearchEngineOptions { // Directory to persist files for Icing. Required. // If Icing was previously initialized with this directory, it will reload @@ -104,6 +104,29 @@ message IcingSearchEngineOptions { // to dynamic trie key mapper). optional bool use_persistent_hash_map = 10; + // Integer index bucket split threshold. + optional int32 integer_index_bucket_split_threshold = 11 [default = 65536]; + + // Whether Icing should sort and merge its lite index HitBuffer unsorted tail + // at indexing time. + // + // If set to true, the HitBuffer will be sorted at indexing time after + // exceeding the sort threshold. If false, the HifBuffer will be sorted at + // querying time, before the first query after inserting new elements into the + // HitBuffer. + // + // The default value is false. + optional bool lite_index_sort_at_indexing = 12; + + // Size (in bytes) at which Icing's lite index should sort and merge the + // HitBuffer's unsorted tail into the sorted head for sorting at indexing + // time. Size specified here is the maximum byte size to allow for the + // unsorted tail section. + // + // Setting a lower sort size reduces querying latency at the expense of + // indexing latency. + optional int32 lite_index_sort_size = 13 [default = 8192]; // 8 KiB + reserved 2; } diff --git a/proto/icing/proto/logging.proto b/proto/icing/proto/logging.proto index ca795cd..418fc88 100644 --- a/proto/icing/proto/logging.proto +++ b/proto/icing/proto/logging.proto @@ -76,7 +76,7 @@ message InitializeStatsProto { // Time used to restore the index. optional int32 index_restoration_latency_ms = 6; - // Time used to restore the index. + // Time used to restore the schema store. optional int32 schema_store_recovery_latency_ms = 7; // Status regarding how much data is lost during the initialization. @@ -117,7 +117,7 @@ message InitializeStatsProto { } // Stats of the top-level function IcingSearchEngine::Put(). -// Next tag: 10 +// Next tag: 11 message PutDocumentStatsProto { // Overall time used for the function call. optional int32 latency_ms = 1; @@ -151,6 +151,9 @@ message PutDocumentStatsProto { // Time used to index all qualified id join strings in the document. optional int32 qualified_id_join_index_latency_ms = 9; + + // Time used to sort and merge the LiteIndex's HitBuffer. + optional int32 lite_index_sort_latency_ms = 10; } // Stats of the top-level function IcingSearchEngine::Search() and diff --git a/proto/icing/proto/schema.proto b/proto/icing/proto/schema.proto index b972ece..c716dba 100644 --- a/proto/icing/proto/schema.proto +++ b/proto/icing/proto/schema.proto @@ -138,15 +138,22 @@ message StringIndexingConfig { } // Describes how a document property should be indexed. -// Next tag: 2 +// Next tag: 3 message DocumentIndexingConfig { // OPTIONAL: Whether nested properties within the document property should be - // indexed. If true, then the nested properties will be indexed according to + // indexed. If true, then all nested properties will be indexed according to // the property's own indexing configurations. If false, nested documents' // properties will not be indexed even if they have an indexing configuration. // // The default value is false. optional bool index_nested_properties = 1; + + // List of nested properties within the document to index. Only the + // provided list of properties will be indexed according to the property's + // indexing configurations. + // + // index_nested_properties must be false in order to use this feature. + repeated string indexable_nested_properties_list = 2; } // Describes how a int64 property should be indexed. diff --git a/proto/icing/proto/search.proto b/proto/icing/proto/search.proto index fca669a..7f4fb3e 100644 --- a/proto/icing/proto/search.proto +++ b/proto/icing/proto/search.proto @@ -27,7 +27,7 @@ option java_multiple_files = true; option objc_class_prefix = "ICNG"; // Client-supplied specifications on what documents to retrieve. -// Next tag: 10 +// Next tag: 11 message SearchSpecProto { // REQUIRED: The "raw" query string that users may type. For example, "cat" // will search for documents with the term cat in it. @@ -102,11 +102,21 @@ message SearchSpecProto { // Finer-grained locks are implemented around code paths that write changes to // Icing during Search. optional bool use_read_only_search = 9 [default = true]; + + // TODO(b/294266822): Handle multiple property filter lists for same schema + // type. + // How to specify a subset of properties to be searched. If no type property + // filter has been specified for a schema type (no TypePropertyMask for the + // given schema type), then *all* properties of that schema type will be + // searched. If an empty property filter is specified for a given schema type + // (TypePropertyMask for the given schema type has empty paths field), no + // properties of that schema type will be searched. + repeated TypePropertyMask type_property_filters = 10; } // Client-supplied specifications on what to include/how to format the search // results. -// Next tag: 9 +// Next tag: 10 message ResultSpecProto { // The results will be returned in pages, and num_per_page specifies the // number of documents in one page. @@ -211,6 +221,18 @@ message ResultSpecProto { // The max # of child documents will be attached and returned in the result // for each parent. It is only used for join API. optional int32 max_joined_children_per_parent_to_return = 8; + + // The max # of results being scored and ranked. + // Running time of ScoringProcessor and Ranker is O(num_to_score) according to + // results of //icing/scoring:score-and-rank_benchmark. Note that + // the process includes scoring, building a heap, and popping results from the + // heap. + // + // 30000 results can be scored and ranked within 3 ms on a Pixel 3 XL + // according to results of + // //icing/scoring:score-and-rank_benchmark, so set it as the + // default value. + optional int32 num_to_score = 9 [default = 30000]; } // The representation of a single match within a DocumentProto property. diff --git a/synced_AOSP_CL_number.txt b/synced_AOSP_CL_number.txt index afb1234..bd3f395 100644 --- a/synced_AOSP_CL_number.txt +++ b/synced_AOSP_CL_number.txt @@ -1 +1 @@ -set(synced_AOSP_CL_number=537223436) +set(synced_AOSP_CL_number=561560020) |