diff options
Diffstat (limited to 'icing/tokenization/raw-query-tokenizer_test.cc')
| -rw-r--r-- | icing/tokenization/raw-query-tokenizer_test.cc | 7 |
1 files changed, 5 insertions, 2 deletions
diff --git a/icing/tokenization/raw-query-tokenizer_test.cc b/icing/tokenization/raw-query-tokenizer_test.cc index a00f2f7..2044f95 100644 --- a/icing/tokenization/raw-query-tokenizer_test.cc +++ b/icing/tokenization/raw-query-tokenizer_test.cc @@ -21,6 +21,7 @@ #include "icing/testing/icu-data-file-helper.h" #include "icing/testing/test-data.h" #include "icing/tokenization/language-segmenter-factory.h" +#include "icing/tokenization/language-segmenter.h" #include "icing/tokenization/tokenizer-factory.h" #include "icing/tokenization/tokenizer.h" #include "unicode/uloc.h" @@ -60,8 +61,10 @@ TEST_F(RawQueryTokenizerTest, NoTokensBeforeAdvancing) { language_segmenter.get())); constexpr std::string_view kText = "Hello, world!"; - ICING_ASSERT_OK_AND_ASSIGN(auto token_iterator, - raw_query_tokenizer->Tokenize(kText)); + ICING_ASSERT_OK_AND_ASSIGN( + auto token_iterator, + raw_query_tokenizer->Tokenize( + kText, LanguageSegmenter::AccessType::kForwardIterator)); // We should get no tokens if we get the token before advancing. EXPECT_THAT(token_iterator->GetTokens(), IsEmpty()); |