Sync from upstream

Descriptions:
==============
Enforce kMaxDocumentId when inserting new documents.
==============
Resets members before reinitializing in IcingSearchEngine Reset()
==============
Exclude a data files for tests under the icing/testdata directory.

Bug: 192464981
Change-Id: I1f8855550791a93022068ea78b6cba2a0758bfb1

30 files changed, 87 insertions, 59 deletions

diff --git a/icing/icing-search-engine.cc b/icing/icing-search-engine.cc
index 55f59f2..1efad9b 100644
--- a/icing/icing-search-engine.cc
+++ b/icing/icing-search-engine.cc
@@ -1705,24 +1705,22 @@ ResetResultProto IcingSearchEngine::Reset() {
   ResetResultProto result_proto;
   StatusProto* result_status = result_proto.mutable_status();
 
-  int64_t before_size = filesystem_->GetDiskUsage(options_.base_dir().c_str());
+  absl_ports::unique_lock l(&mutex_);
 
-  if (!filesystem_->DeleteDirectoryRecursively(options_.base_dir().c_str())) {
-    int64_t after_size = filesystem_->GetDiskUsage(options_.base_dir().c_str());
-    if (after_size != before_size) {
-      // Our filesystem doesn't atomically delete. If we have a discrepancy in
-      // size, then that means we may have deleted some files, but not others.
-      // So our data is in an invalid state now.
-      result_status->set_code(StatusProto::INTERNAL);
-      return result_proto;
-    }
+  initialized_ = false;
 
-    result_status->set_code(StatusProto::ABORTED);
+  // Resets members variables
+  schema_store_.reset();
+  document_store_.reset();
+  language_segmenter_.reset();
+  normalizer_.reset();
+  index_.reset();
+
+  if (!filesystem_->DeleteDirectoryRecursively(options_.base_dir().c_str())) {
+    result_status->set_code(StatusProto::INTERNAL);
     return result_proto;
   }
 
-  absl_ports::unique_lock l(&mutex_);
-  initialized_ = false;
   if (InternalInitialize().status().code() != StatusProto::OK) {
     // We shouldn't hit the following Initialize errors:
     //   NOT_FOUND: all data was cleared, we aren't expecting anything
diff --git a/icing/icing-search-engine.h b/icing/icing-search-engine.h
index 3dc7e29..855401f 100644
--- a/icing/icing-search-engine.h
+++ b/icing/icing-search-engine.h
@@ -178,6 +178,7 @@ class IcingSearchEngine {
   //
   // Returns:
   //   OK on success
+  //   OUT_OF_SPACE if exceeds maximum number of allowed documents
   //   FAILED_PRECONDITION if a schema has not been set yet, IcingSearchEngine
   //     has not been initialized yet.
   //   NOT_FOUND if there is no SchemaTypeConfig in the SchemaProto that matches
diff --git a/icing/icing-search-engine_benchmark.cc b/icing/icing-search-engine_benchmark.cc
index 316b74f..ba9aed1 100644
--- a/icing/icing-search-engine_benchmark.cc
+++ b/icing/icing-search-engine_benchmark.cc
@@ -70,6 +70,7 @@ namespace lib {
 namespace {
 
 using ::testing::Eq;
+using ::testing::HasSubstr;
 
 // Icing GMSCore has, on average, 17 corpora on a device and 30 corpora at the
 // 95th pct. Most clients use a single type. This is a function of Icing's
@@ -691,6 +692,59 @@ void BM_Delete(benchmark::State& state) {
 }
 BENCHMARK(BM_Delete);
 
+void BM_PutMaxAllowedDocuments(benchmark::State& state) {
+  // Initialize the filesystem
+  std::string test_dir = GetTestTempDir() + "/icing/benchmark";
+  Filesystem filesystem;
+  DestructibleDirectory ddir(filesystem, test_dir);
+
+  // Create the schema.
+  SchemaProto schema =
+      SchemaBuilder()
+          .AddType(SchemaTypeConfigBuilder().SetType("Message").AddProperty(
+              PropertyConfigBuilder()
+                  .SetName("body")
+                  .SetDataTypeString(TermMatchType::PREFIX,
+                                     StringIndexingConfig::TokenizerType::PLAIN)
+                  .SetCardinality(PropertyConfigProto::Cardinality::OPTIONAL)))
+          .Build();
+
+  // Create the index.
+  IcingSearchEngineOptions options;
+  options.set_base_dir(test_dir);
+  options.set_index_merge_size(kIcingFullIndexSize);
+  std::unique_ptr<IcingSearchEngine> icing =
+      std::make_unique<IcingSearchEngine>(options);
+
+  ASSERT_THAT(icing->Initialize().status(), ProtoIsOk());
+  ASSERT_THAT(icing->SetSchema(schema).status(), ProtoIsOk());
+
+  // Create a document that has the term "foo"
+  DocumentProto base_document = DocumentBuilder()
+                                    .SetSchema("Message")
+                                    .SetNamespace("namespace")
+                                    .AddStringProperty("body", "foo")
+                                    .Build();
+
+  // Insert a lot of documents with the term "foo"
+  for (auto s : state) {
+    for (int64_t i = 0; i <= kMaxDocumentId; ++i) {
+      DocumentProto document =
+          DocumentBuilder(base_document).SetUri(std::to_string(i)).Build();
+      EXPECT_THAT(icing->Put(document).status(), ProtoIsOk());
+    }
+  }
+
+  DocumentProto document =
+      DocumentBuilder(base_document).SetUri("out_of_space_uri").Build();
+  PutResultProto put_result_proto = icing->Put(document);
+  EXPECT_THAT(put_result_proto.status(),
+              ProtoStatusIs(StatusProto::OUT_OF_SPACE));
+  EXPECT_THAT(put_result_proto.status().message(),
+              HasSubstr("Exceeded maximum number of documents"));
+}
+BENCHMARK(BM_PutMaxAllowedDocuments);
+
 }  // namespace
 
 }  // namespace lib
diff --git a/icing/icing-search-engine_test.cc b/icing/icing-search-engine_test.cc
index 5846e8b..1ee2d63 100644
--- a/icing/icing-search-engine_test.cc
+++ b/icing/icing-search-engine_test.cc
@@ -5774,11 +5774,11 @@ TEST_F(IcingSearchEngineTest, ResetOk) {
   EXPECT_THAT(icing.SetSchema(empty_schema).status(), ProtoIsOk());
 }
 
-TEST_F(IcingSearchEngineTest, ResetDeleteFailureCausesAbortedError) {
+TEST_F(IcingSearchEngineTest, ResetDeleteFailureCausesInternalError) {
   auto mock_filesystem = std::make_unique<MockFilesystem>();
 
-  // This fails IcingSearchEngine::Reset(). But since we didn't actually delete
-  // anything, we'll be able to consider this just an ABORTED call.
+  // This fails IcingSearchEngine::Reset() with status code INTERNAL and leaves
+  // the IcingSearchEngine instance in an uninitialized state.
   ON_CALL(*mock_filesystem,
           DeleteDirectoryRecursively(StrEq(GetTestBaseDir().c_str())))
       .WillByDefault(Return(false));
@@ -5792,51 +5792,17 @@ TEST_F(IcingSearchEngineTest, ResetDeleteFailureCausesAbortedError) {
 
   DocumentProto document = CreateMessageDocument("namespace", "uri");
   ASSERT_THAT(icing.Put(document).status(), ProtoIsOk());
-  EXPECT_THAT(icing.Reset().status(), ProtoStatusIs(StatusProto::ABORTED));
+  EXPECT_THAT(icing.Reset().status(), ProtoStatusIs(StatusProto::INTERNAL));
 
-  // Everything is still intact.
-  // Can get old data.
   GetResultProto expected_get_result_proto;
-  expected_get_result_proto.mutable_status()->set_code(StatusProto::OK);
+  expected_get_result_proto.mutable_status()->set_code(
+      StatusProto::FAILED_PRECONDITION);
   *expected_get_result_proto.mutable_document() = document;
-  EXPECT_THAT(icing.Get(document.namespace_(), document.uri(),
-                        GetResultSpecProto::default_instance()),
-              EqualsProto(expected_get_result_proto));
-
-  // Can add new data.
-  EXPECT_THAT(icing.Put(CreateMessageDocument("namespace", "uri")).status(),
-              ProtoIsOk());
-}
-
-TEST_F(IcingSearchEngineTest, ResetCreateFailureCausesInternalError) {
-  auto mock_filesystem = std::make_unique<MockFilesystem>();
-
-  // Let all other delete directory calls succeed.
-  EXPECT_CALL(*mock_filesystem,
-              DeleteDirectoryRecursively(Matcher<const char*>(_)))
-      .WillRepeatedly(Return(true));
-
-  // This prevents IcingSearchEngine from deleting our base dir when resetting
-  EXPECT_CALL(*mock_filesystem, DeleteDirectoryRecursively(Matcher<const char*>(
-                                    StrEq(GetTestBaseDir().c_str()))))
-      .WillOnce(Return(false));
-
-  // The first call will show our base directory had 100 bytes, but after we
-  // falied to delete, we lost those 100 bytes. So this will be reported as an
-  // INTERNAL error since data was lost.
-  EXPECT_CALL(
-      *mock_filesystem,
-      GetDiskUsage(Matcher<const char*>(StrEq(GetTestBaseDir().c_str()))))
-      .WillOnce(Return(100))
-      .WillOnce(Return(0));
-
-  TestIcingSearchEngine icing(GetDefaultIcingOptions(),
-                              std::move(mock_filesystem),
-                              std::make_unique<IcingFilesystem>(),
-                              std::make_unique<FakeClock>(), GetTestJniCache());
-  ASSERT_THAT(icing.Initialize().status(), ProtoIsOk());
-  ASSERT_THAT(icing.SetSchema(CreateMessageSchema()).status(), ProtoIsOk());
-  EXPECT_THAT(icing.Reset().status(), ProtoStatusIs(StatusProto::INTERNAL));
+  EXPECT_THAT(icing
+                  .Get(document.namespace_(), document.uri(),
+                       GetResultSpecProto::default_instance())
+                  .status(),
+              ProtoStatusIs(StatusProto::FAILED_PRECONDITION));
 }
 
 TEST_F(IcingSearchEngineTest, SnippetNormalization) {
diff --git a/icing/store/document-store.cc b/icing/store/document-store.cc
index fc797b4..6479b97 100644
--- a/icing/store/document-store.cc
+++ b/icing/store/document-store.cc
@@ -74,7 +74,9 @@ constexpr char kNamespaceMapperFilename[] = "namespace_mapper";
 constexpr char kUsageStoreDirectoryName[] = "usage_store";
 constexpr char kCorpusIdMapperFilename[] = "corpus_mapper";
 
-constexpr int32_t kUriMapperMaxSize = 12 * 1024 * 1024;  // 12 MiB
+// Determined through manual testing to allow for 1 million uris. 1 million
+// because we allow up to 1 million DocumentIds.
+constexpr int32_t kUriMapperMaxSize = 36 * 1024 * 1024;  // 36 MiB
 
 // 384 KiB for a KeyMapper would allow each internal array to have a max of
 // 128 KiB for storage.
@@ -806,6 +808,12 @@ libtextclassifier3::StatusOr<DocumentId> DocumentStore::InternalPut(
 
   // Creates a new document id, updates key mapper and document_id mapper
   DocumentId new_document_id = document_id_mapper_->num_elements();
+  if (!IsDocumentIdValid(new_document_id)) {
+    return absl_ports::ResourceExhaustedError(
+        "Exceeded maximum number of documents. Try calling Optimize to reclaim "
+        "some space.");
+  }
+
   ICING_RETURN_IF_ERROR(document_key_mapper_->Put(
       MakeFingerprint(name_space, uri), new_document_id));
   ICING_RETURN_IF_ERROR(document_id_mapper_->Set(new_document_id, file_offset));
diff --git a/icing/store/document-store.h b/icing/store/document-store.h
index 79d99d4..a60aab1 100644
--- a/icing/store/document-store.h
+++ b/icing/store/document-store.h
@@ -156,6 +156,7 @@ class DocumentStore {
   //
   // Returns:
   //   A newly generated document id on success
+  //   RESOURCE_EXHAUSED if exceeds maximum number of allowed documents
   //   FAILED_PRECONDITION if schema hasn't been set yet
   //   NOT_FOUND if the schema_type or a property config of the document doesn't
   //     exist in schema
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_arm/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_android_arm/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 67a5bbd..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_arm/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_arm/document_dir/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_android_arm/document_dir/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 70ec244..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_arm/document_dir/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_arm/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_android_arm/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 6177f48..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_arm/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_arm/index_dir/idx/lite.lexicon.h b/icing/testdata/not_portable_log/icing_search_engine_android_arm/index_dir/idx/lite.lexicon.h
deleted file mode 100644
index 624444e..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_arm/index_dir/idx/lite.lexicon.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_arm/index_dir/idx/main/main-lexicon.h b/icing/testdata/not_portable_log/icing_search_engine_android_arm/index_dir/idx/main/main-lexicon.h
deleted file mode 100644
index 82d6712..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_arm/index_dir/idx/main/main-lexicon.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_arm/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_android_arm/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index d1b1cc1..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_arm/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_x86/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_android_x86/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 67a5bbd..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_x86/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_x86/document_dir/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_android_x86/document_dir/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 70ec244..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_x86/document_dir/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_x86/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_android_x86/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 6177f48..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_x86/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_x86/index_dir/idx/lite.lexicon.h b/icing/testdata/not_portable_log/icing_search_engine_android_x86/index_dir/idx/lite.lexicon.h
deleted file mode 100644
index 624444e..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_x86/index_dir/idx/lite.lexicon.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_x86/index_dir/idx/main/main-lexicon.h b/icing/testdata/not_portable_log/icing_search_engine_android_x86/index_dir/idx/main/main-lexicon.h
deleted file mode 100644
index 82d6712..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_x86/index_dir/idx/main/main-lexicon.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_android_x86/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_android_x86/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index d1b1cc1..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_android_x86/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_ios/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_ios/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 67a5bbd..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_ios/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_ios/document_dir/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_ios/document_dir/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 70ec244..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_ios/document_dir/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_ios/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_ios/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 6177f48..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_ios/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_ios/index_dir/idx/lite.lexicon.h b/icing/testdata/not_portable_log/icing_search_engine_ios/index_dir/idx/lite.lexicon.h
deleted file mode 100644
index 624444e..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_ios/index_dir/idx/lite.lexicon.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_ios/index_dir/idx/main/main-lexicon.h b/icing/testdata/not_portable_log/icing_search_engine_ios/index_dir/idx/main/main-lexicon.h
deleted file mode 100644
index 82d6712..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_ios/index_dir/idx/main/main-lexicon.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_ios/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_ios/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index d1b1cc1..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_ios/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_linux/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_linux/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 67a5bbd..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_linux/document_dir/corpus_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_linux/document_dir/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_linux/document_dir/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 70ec244..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_linux/document_dir/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_linux/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_linux/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index 6177f48..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_linux/document_dir/namespace_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_linux/index_dir/idx/lite.lexicon.h b/icing/testdata/not_portable_log/icing_search_engine_linux/index_dir/idx/lite.lexicon.h
deleted file mode 100644
index c95dba8..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_linux/index_dir/idx/lite.lexicon.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_linux/index_dir/idx/main/main-lexicon.h b/icing/testdata/not_portable_log/icing_search_engine_linux/index_dir/idx/main/main-lexicon.h
deleted file mode 100644
index 6b87b9c..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_linux/index_dir/idx/main/main-lexicon.h
+++ /dev/null
diff --git a/icing/testdata/not_portable_log/icing_search_engine_linux/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h b/icing/testdata/not_portable_log/icing_search_engine_linux/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h
deleted file mode 100644
index d1b1cc1..0000000
--- a/icing/testdata/not_portable_log/icing_search_engine_linux/schema_dir/schema_type_mapper/key_mapper_dir/key_mapper.h
+++ /dev/null