Sync from upstream.

Descriptions:
======================================================================
[FileBackedVector Consolidation][4/x] Fix potential PWrite bug in GrowIfNecessary
======================================================================
[FileBackedVector Consolidation][5/x] Create benchmark for FileBackedVector
======================================================================
[FileBackedVector Consolidation][6/x] Avoid calling GetFileSize in GrowIfNecessary
======================================================================
[PersistentHashMap][3.3/x] Implement Delete
======================================================================
Fix the PopulateMatchedTermsStats bug
======================================================================
Add JNI latency for query latency stats breakdown.
======================================================================
[ResultStateManager] Thread safety test1
======================================================================
[ResultStateManager][2/x] Thread safety test2
======================================================================
Add native lock contention latency for measuring query latency
======================================================================
Fix implementation of HasMember operator in ANTLR-based list-filter prototype.
======================================================================
Fix improper uses of std::string_view
======================================================================
Extend the scale of Icing
======================================================================
Decouple the term frequency array from DocHitInfo
======================================================================
Disable hit_term_frequency for non-relevance queries
======================================================================
[ResultStateManager][3/x] Thread safety test3
======================================================================
[PersistentHashMap][4/x] Implement iterator
=======================================================================
Fix the lite index compaction bug
=======================================================================

Change-Id: I0edad67affed97af107e2d7cd73770e0268c0903

1 files changed, 45 insertions, 44 deletions

diff --git a/icing/index/iterator/doc-hit-info-iterator-or_test.cc b/icing/index/iterator/doc-hit-info-iterator-or_test.cc
index 3f00a39..6e6872c 100644
--- a/icing/index/iterator/doc-hit-info-iterator-or_test.cc
+++ b/icing/index/iterator/doc-hit-info-iterator-or_test.cc
@@ -183,24 +183,24 @@ TEST(DocHitInfoIteratorOrTest, PopulateMatchedTermsStats) {
     // Arbitrary section ids for the documents in the DocHitInfoIterators.
     // Created to test correct section_id_mask behavior.
     SectionIdMask section_id_mask1 = 0b01010101;  // hits in sections 0, 2, 4, 6
-    std::array<Hit::TermFrequency, kMaxSectionId> term_frequencies1{
-        1, 0, 2, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0};
+    std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies1{
+        1, 0, 2, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0};
     SectionIdMask section_id_mask2 = 0b00000110;  // hits in sections 1, 2
-    std::array<Hit::TermFrequency, kMaxSectionId> term_frequencies2{
-        0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies2{
+        0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
-    DocHitInfo doc_hit_info1 = DocHitInfo(4);
+    DocHitInfoTermFrequencyPair doc_hit_info1 = DocHitInfo(4);
     doc_hit_info1.UpdateSection(/*section_id=*/0, /*hit_term_frequency=*/1);
     doc_hit_info1.UpdateSection(/*section_id=*/2, /*hit_term_frequency=*/2);
     doc_hit_info1.UpdateSection(/*section_id=*/4, /*hit_term_frequency=*/3);
     doc_hit_info1.UpdateSection(/*section_id=*/6, /*hit_term_frequency=*/4);
 
-    DocHitInfo doc_hit_info2 = DocHitInfo(4);
+    DocHitInfoTermFrequencyPair doc_hit_info2 = DocHitInfo(4);
     doc_hit_info2.UpdateSection(/*section_id=*/1, /*hit_term_frequency=*/2);
     doc_hit_info2.UpdateSection(/*section_id=*/2, /*hit_term_frequency=*/6);
 
-    std::vector<DocHitInfo> first_vector = {doc_hit_info1};
-    std::vector<DocHitInfo> second_vector = {doc_hit_info2};
+    std::vector<DocHitInfoTermFrequencyPair> first_vector = {doc_hit_info1};
+    std::vector<DocHitInfoTermFrequencyPair> second_vector = {doc_hit_info2};
 
     auto first_iter =
         std::make_unique<DocHitInfoIteratorDummy>(first_vector, "hi");
@@ -235,15 +235,15 @@ TEST(DocHitInfoIteratorOrTest, PopulateMatchedTermsStats) {
     // Arbitrary section ids for the documents in the DocHitInfoIterators.
     // Created to test correct section_id_mask behavior.
     SectionIdMask section_id_mask1 = 0b00000101;  // hits in sections 0, 2
-    std::array<Hit::TermFrequency, kMaxSectionId> term_frequencies1{
-        1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies1{
+        1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
-    DocHitInfo doc_hit_info1 = DocHitInfo(4);
+    DocHitInfoTermFrequencyPair doc_hit_info1 = DocHitInfo(4);
     doc_hit_info1.UpdateSection(/*section_id=*/0, /*hit_term_frequency=*/1);
     doc_hit_info1.UpdateSection(/*section_id=*/2, /*hit_term_frequency=*/2);
 
-    std::vector<DocHitInfo> first_vector = {doc_hit_info1};
-    std::vector<DocHitInfo> second_vector = {doc_hit_info1};
+    std::vector<DocHitInfoTermFrequencyPair> first_vector = {doc_hit_info1};
+    std::vector<DocHitInfoTermFrequencyPair> second_vector = {doc_hit_info1};
 
     auto first_iter =
         std::make_unique<DocHitInfoIteratorDummy>(first_vector, "hi");
@@ -274,24 +274,24 @@ TEST(DocHitInfoIteratorOrTest, PopulateMatchedTermsStats) {
     // Arbitrary section ids for the documents in the DocHitInfoIterators.
     // Created to test correct section_id_mask behavior.
     SectionIdMask section_id_mask1 = 0b01010101;  // hits in sections 0, 2, 4, 6
-    std::array<Hit::TermFrequency, kMaxSectionId> term_frequencies1{
-        1, 0, 2, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0};
+    std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies1{
+        1, 0, 2, 0, 3, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0};
     SectionIdMask section_id_mask2 = 0b00000110;  // hits in sections 1, 2
-    std::array<Hit::TermFrequency, kMaxSectionId> term_frequencies2{
-        0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+    std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies2{
+        0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
 
-    DocHitInfo doc_hit_info1 = DocHitInfo(4);
+    DocHitInfoTermFrequencyPair doc_hit_info1 = DocHitInfo(4);
     doc_hit_info1.UpdateSection(/*section_id=*/0, /*hit_term_frequency=*/1);
     doc_hit_info1.UpdateSection(/*section_id=*/2, /*hit_term_frequency=*/2);
     doc_hit_info1.UpdateSection(/*section_id=*/4, /*hit_term_frequency=*/3);
     doc_hit_info1.UpdateSection(/*section_id=*/6, /*hit_term_frequency=*/4);
 
-    DocHitInfo doc_hit_info2 = DocHitInfo(5);
+    DocHitInfoTermFrequencyPair doc_hit_info2 = DocHitInfo(5);
     doc_hit_info2.UpdateSection(/*section_id=*/1, /*hit_term_frequency=*/2);
     doc_hit_info2.UpdateSection(/*section_id=*/2, /*hit_term_frequency=*/6);
 
-    std::vector<DocHitInfo> first_vector = {doc_hit_info1};
-    std::vector<DocHitInfo> second_vector = {doc_hit_info2};
+    std::vector<DocHitInfoTermFrequencyPair> first_vector = {doc_hit_info1};
+    std::vector<DocHitInfoTermFrequencyPair> second_vector = {doc_hit_info2};
 
     auto first_iter =
         std::make_unique<DocHitInfoIteratorDummy>(first_vector, "hi");
@@ -477,55 +477,56 @@ TEST(DocHitInfoIteratorOrNaryTest, PopulateMatchedTermsStats) {
   // DocHitInfoIterators.
   // For term "hi", document 10 and 8
   SectionIdMask section_id_mask1_hi = 0b01000101;  // hits in sections 0, 2, 6
-  std::array<Hit::TermFrequency, kMaxSectionId> term_frequencies1_hi{
-      1, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0};
-  DocHitInfo doc_hit_info1_hi = DocHitInfo(10);
+  std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies1_hi{
+      1, 0, 2, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+  DocHitInfoTermFrequencyPair doc_hit_info1_hi = DocHitInfo(10);
   doc_hit_info1_hi.UpdateSection(/*section_id=*/0, /*hit_term_frequency=*/1);
   doc_hit_info1_hi.UpdateSection(/*section_id=*/2, /*hit_term_frequency=*/2);
   doc_hit_info1_hi.UpdateSection(/*section_id=*/6, /*hit_term_frequency=*/4);
 
   SectionIdMask section_id_mask2_hi = 0b00000110;  // hits in sections 1, 2
-  std::array<Hit::TermFrequency, kMaxSectionId> term_frequencies2_hi{
-      0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-  DocHitInfo doc_hit_info2_hi = DocHitInfo(8);
+  std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies2_hi{
+      0, 2, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+  DocHitInfoTermFrequencyPair doc_hit_info2_hi = DocHitInfo(8);
   doc_hit_info2_hi.UpdateSection(/*section_id=*/1, /*hit_term_frequency=*/2);
   doc_hit_info2_hi.UpdateSection(/*section_id=*/2, /*hit_term_frequency=*/6);
 
   // For term "hello", document 10 and 9
   SectionIdMask section_id_mask1_hello = 0b00001001;  // hits in sections 0, 3
-  std::array<Hit::TermFrequency, kMaxSectionId> term_frequencies1_hello{
-      2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-  DocHitInfo doc_hit_info1_hello = DocHitInfo(10);
+  std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies1_hello{
+      2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+  DocHitInfoTermFrequencyPair doc_hit_info1_hello = DocHitInfo(10);
   doc_hit_info1_hello.UpdateSection(/*section_id=*/0, /*hit_term_frequency=*/2);
   doc_hit_info1_hello.UpdateSection(/*section_id=*/3, /*hit_term_frequency=*/3);
 
   SectionIdMask section_id_mask2_hello = 0b00001100;  // hits in sections 2, 3
-  std::array<Hit::TermFrequency, kMaxSectionId> term_frequencies2_hello{
-      0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-  DocHitInfo doc_hit_info2_hello = DocHitInfo(9);
+  std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies2_hello{
+      0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+  DocHitInfoTermFrequencyPair doc_hit_info2_hello = DocHitInfo(9);
   doc_hit_info2_hello.UpdateSection(/*section_id=*/2, /*hit_term_frequency=*/3);
   doc_hit_info2_hello.UpdateSection(/*section_id=*/3, /*hit_term_frequency=*/2);
 
   // For term "ciao", document 9 and 8
   SectionIdMask section_id_mask1_ciao = 0b00000011;  // hits in sections 0, 1
-  std::array<Hit::TermFrequency, kMaxSectionId> term_frequencies1_ciao{
-      2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-  DocHitInfo doc_hit_info1_ciao = DocHitInfo(9);
+  std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies1_ciao{
+      2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+  DocHitInfoTermFrequencyPair doc_hit_info1_ciao = DocHitInfo(9);
   doc_hit_info1_ciao.UpdateSection(/*section_id=*/0, /*hit_term_frequency=*/2);
   doc_hit_info1_ciao.UpdateSection(/*section_id=*/1, /*hit_term_frequency=*/3);
 
   SectionIdMask section_id_mask2_ciao = 0b00011000;  // hits in sections 3, 4
-  std::array<Hit::TermFrequency, kMaxSectionId> term_frequencies2_ciao{
-      0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
-  DocHitInfo doc_hit_info2_ciao = DocHitInfo(8);
+  std::array<Hit::TermFrequency, kTotalNumSections> term_frequencies2_ciao{
+      0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
+  DocHitInfoTermFrequencyPair doc_hit_info2_ciao = DocHitInfo(8);
   doc_hit_info2_ciao.UpdateSection(/*section_id=*/3, /*hit_term_frequency=*/3);
   doc_hit_info2_ciao.UpdateSection(/*section_id=*/4, /*hit_term_frequency=*/2);
 
-  std::vector<DocHitInfo> first_vector = {doc_hit_info1_hi, doc_hit_info2_hi};
-  std::vector<DocHitInfo> second_vector = {doc_hit_info1_hello,
-                                           doc_hit_info2_hello};
-  std::vector<DocHitInfo> third_vector = {doc_hit_info1_ciao,
-                                          doc_hit_info2_ciao};
+  std::vector<DocHitInfoTermFrequencyPair> first_vector = {doc_hit_info1_hi,
+                                                           doc_hit_info2_hi};
+  std::vector<DocHitInfoTermFrequencyPair> second_vector = {
+      doc_hit_info1_hello, doc_hit_info2_hello};
+  std::vector<DocHitInfoTermFrequencyPair> third_vector = {doc_hit_info1_ciao,
+                                                           doc_hit_info2_ciao};
 
   auto first_iter =
       std::make_unique<DocHitInfoIteratorDummy>(first_vector, "hi");