diff options
Diffstat (limited to 'icing/util/tokenized-document.cc')
| -rw-r--r-- | icing/util/tokenized-document.cc | 6 |
1 files changed, 4 insertions, 2 deletions
diff --git a/icing/util/tokenized-document.cc b/icing/util/tokenized-document.cc index facb267..1c11c3c 100644 --- a/icing/util/tokenized-document.cc +++ b/icing/util/tokenized-document.cc @@ -44,8 +44,10 @@ libtextclassifier3::StatusOr<std::vector<TokenizedSection>> Tokenize( section.metadata.tokenizer, language_segmenter)); std::vector<std::string_view> token_sequence; for (std::string_view subcontent : section.content) { - ICING_ASSIGN_OR_RETURN(std::unique_ptr<Tokenizer::Iterator> itr, - tokenizer->Tokenize(subcontent)); + ICING_ASSIGN_OR_RETURN( + std::unique_ptr<Tokenizer::Iterator> itr, + tokenizer->Tokenize(subcontent, + LanguageSegmenter::AccessType::kForwardIterator)); while (itr->Advance()) { std::vector<Token> batch_tokens = itr->GetTokens(); for (const Token& token : batch_tokens) { |