aboutsummaryrefslogtreecommitdiff
path: root/icing/tokenization/reverse_jni/reverse-jni-language-segmenter_test.cc
diff options
context:
space:
mode:
Diffstat (limited to 'icing/tokenization/reverse_jni/reverse-jni-language-segmenter_test.cc')
-rw-r--r--icing/tokenization/reverse_jni/reverse-jni-language-segmenter_test.cc184
1 files changed, 60 insertions, 124 deletions
diff --git a/icing/tokenization/reverse_jni/reverse-jni-language-segmenter_test.cc b/icing/tokenization/reverse_jni/reverse-jni-language-segmenter_test.cc
index be652ff..47a01fe 100644
--- a/icing/tokenization/reverse_jni/reverse-jni-language-segmenter_test.cc
+++ b/icing/tokenization/reverse_jni/reverse-jni-language-segmenter_test.cc
@@ -394,10 +394,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, ContinuousWhitespaces) {
// iterator is done.
text_with_spaces = absl_ports::StrCat(std::string(kNumSeparators, ' '),
"Hello", " ", "World");
- ICING_ASSERT_OK_AND_ASSIGN(
- auto itr,
- language_segmenter->Segment(
- text_with_spaces, LanguageSegmenter::AccessType::kForwardIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(auto itr,
+ language_segmenter->Segment(text_with_spaces));
std::vector<std::string_view> terms;
while (itr->Advance()) {
terms.push_back(itr->GetTerm());
@@ -493,10 +491,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, ResetToStartUtf32WordConnector) {
auto segmenter, language_segmenter_factory::Create(
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
constexpr std::string_view kText = "com:google:android is package";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ segmenter->Segment(kText));
// String: "com:google:android is package"
// ^ ^^ ^^
@@ -512,10 +508,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, NewIteratorResetToStartUtf32) {
auto segmenter, language_segmenter_factory::Create(
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
constexpr std::string_view kText = "How are you你好吗お元気ですか";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ segmenter->Segment(kText));
// String: "How are you你好吗お元気ですか"
// ^ ^^ ^^ ^ ^ ^ ^ ^ ^
@@ -530,10 +524,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, IteratorOneAdvanceResetToStartUtf32) {
auto segmenter, language_segmenter_factory::Create(
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
constexpr std::string_view kText = "How are you你好吗お元気ですか";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ segmenter->Segment(kText));
// String: "How are you你好吗お元気ですか"
// ^ ^^ ^^ ^ ^ ^ ^ ^ ^
@@ -550,10 +542,8 @@ TEST_P(ReverseJniLanguageSegmenterTest,
auto segmenter, language_segmenter_factory::Create(
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
constexpr std::string_view kText = "How are you你好吗お元気ですか";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ segmenter->Segment(kText));
// String: "How are you你好吗お元気ですか"
// ^ ^^ ^^ ^ ^ ^ ^ ^ ^
@@ -572,10 +562,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, IteratorDoneResetToStartUtf32) {
auto segmenter, language_segmenter_factory::Create(
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
constexpr std::string_view kText = "How are you你好吗お元気ですか";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ segmenter->Segment(kText));
// String: "How are you你好吗お元気ですか"
// ^ ^^ ^^ ^ ^ ^ ^ ^ ^
@@ -593,10 +581,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, ResetToTermAfterUtf32WordConnector) {
auto segmenter, language_segmenter_factory::Create(
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
constexpr std::string_view kText = "package com:google:android name";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ segmenter->Segment(kText));
// String: "package com:google:android name"
// ^ ^^ ^^
@@ -618,10 +604,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, ResetToTermAfterUtf32OutOfBounds) {
auto segmenter, language_segmenter_factory::Create(
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
constexpr std::string_view kText = "How are you你好吗お元気ですか";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ segmenter->Segment(kText));
// String: "How are you你好吗お元気ですか"
// ^ ^^ ^^ ^ ^ ^ ^ ^ ^
@@ -651,15 +635,13 @@ TEST_P(ReverseJniLanguageSegmenterTest,
constexpr std::string_view kText = "How are𡔖 you你好吗お元気ですか";
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_itr,
- segmenter->Segment(kText,
- LanguageSegmenter::AccessType::kForwardIterator));
+ segmenter->Segment(kText));
std::vector<std::string_view> advance_terms =
GetAllTermsAdvance(advance_itr.get());
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ segmenter->Segment(kText));
std::vector<std::string_view> reset_terms =
GetAllTermsResetAfterUtf32(reset_to_term_itr.get());
@@ -675,15 +657,13 @@ TEST_P(ReverseJniLanguageSegmenterTest,
constexpr std::string_view kThai = "ฉันเดินไปทำงานทุกวัน";
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_itr,
- segmenter->Segment(kThai,
- LanguageSegmenter::AccessType::kForwardIterator));
+ segmenter->Segment(kThai));
std::vector<std::string_view> advance_terms =
GetAllTermsAdvance(advance_itr.get());
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr,
- segmenter->Segment(
- kThai, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ segmenter->Segment(kThai));
std::vector<std::string_view> reset_terms =
GetAllTermsResetAfterUtf32(reset_to_term_itr.get());
@@ -699,15 +679,13 @@ TEST_P(ReverseJniLanguageSegmenterTest,
constexpr std::string_view kKorean = "나는 매일 출근합니다.";
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_itr,
- segmenter->Segment(kKorean,
- LanguageSegmenter::AccessType::kForwardIterator));
+ segmenter->Segment(kKorean));
std::vector<std::string_view> advance_terms =
GetAllTermsAdvance(advance_itr.get());
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr,
- segmenter->Segment(
- kKorean, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ segmenter->Segment(kKorean));
std::vector<std::string_view> reset_terms =
GetAllTermsResetAfterUtf32(reset_to_term_itr.get());
@@ -727,15 +705,13 @@ TEST_P(ReverseJniLanguageSegmenterTest,
constexpr std::string_view kText = "How are𡔖 you你好吗お元気ですか";
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_itr,
- segmenter->Segment(kText,
- LanguageSegmenter::AccessType::kForwardIterator));
+ segmenter->Segment(kText));
std::vector<std::string_view> advance_terms =
GetAllTermsAdvance(advance_itr.get());
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_and_reset_itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ segmenter->Segment(kText));
std::vector<std::string_view> advance_and_reset_terms =
GetAllTermsAdvanceAndResetAfterUtf32(advance_and_reset_itr.get());
@@ -752,15 +728,13 @@ TEST_P(ReverseJniLanguageSegmenterTest,
constexpr std::string_view kThai = "ฉันเดินไปทำงานทุกวัน";
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_itr,
- segmenter->Segment(kThai,
- LanguageSegmenter::AccessType::kForwardIterator));
+ segmenter->Segment(kThai));
std::vector<std::string_view> advance_terms =
GetAllTermsAdvance(advance_itr.get());
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_and_reset_itr,
- segmenter->Segment(
- kThai, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ segmenter->Segment(kThai));
std::vector<std::string_view> advance_and_reset_terms =
GetAllTermsAdvanceAndResetAfterUtf32(advance_and_reset_itr.get());
@@ -777,15 +751,13 @@ TEST_P(ReverseJniLanguageSegmenterTest,
constexpr std::string_view kKorean = "나는 매일 출근합니다.";
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_itr,
- segmenter->Segment(kKorean,
- LanguageSegmenter::AccessType::kForwardIterator));
+ segmenter->Segment(kKorean));
std::vector<std::string_view> advance_terms =
GetAllTermsAdvance(advance_itr.get());
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_and_reset_itr,
- segmenter->Segment(
- kKorean, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ segmenter->Segment(kKorean));
std::vector<std::string_view> advance_and_reset_terms =
GetAllTermsAdvanceAndResetAfterUtf32(advance_and_reset_itr.get());
@@ -801,9 +773,7 @@ TEST_P(ReverseJniLanguageSegmenterTest, MixedLanguagesResetToTermAfterUtf32) {
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- "How are you你好吗お元気ですか",
- LanguageSegmenter::AccessType::kBidirectionalIterator));
+ language_segmenter->Segment("How are you你好吗お元気ですか"));
// String: "How are you你好吗お元気ですか"
// ^ ^^ ^^ ^ ^ ^ ^ ^ ^
@@ -840,11 +810,8 @@ TEST_P(ReverseJniLanguageSegmenterTest,
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
// Multiple continuous whitespaces are treated as one.
constexpr std::string_view kTextWithSpace = "Hello World";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- kTextWithSpace,
- LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ language_segmenter->Segment(kTextWithSpace));
// String: "Hello World"
// ^ ^ ^
@@ -883,10 +850,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, ChineseResetToTermAfterUtf32) {
// CJKT (Chinese, Japanese, Khmer, Thai) are the 4 main languages that
// don't have whitespaces as word delimiter. Chinese
constexpr std::string_view kChinese = "我每天走路去上班。";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- kChinese, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ language_segmenter->Segment(kChinese));
// String: "我每天走路去上班。"
// ^ ^ ^ ^^ ^
// UTF-8 idx: 0 3 9 15 18 24
@@ -912,10 +877,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, JapaneseResetToTermAfterUtf32) {
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
// Japanese
constexpr std::string_view kJapanese = "私は毎日仕事に歩いています。";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- kJapanese, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ language_segmenter->Segment(kJapanese));
// String: "私は毎日仕事に歩いています。"
// ^ ^ ^ ^ ^ ^ ^ ^ ^ ^
// UTF-8 idx: 0 3 6 12 18212427 33 39
@@ -940,10 +903,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, KhmerResetToTermAfterUtf32) {
language_segmenter_factory::Create(
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
constexpr std::string_view kKhmer = "ញុំដើរទៅធ្វើការរាល់ថ្ងៃ។";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- kKhmer, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ language_segmenter->Segment(kKhmer));
// String: "ញុំដើរទៅធ្វើការរាល់ថ្ងៃ។"
// ^ ^ ^ ^ ^
// UTF-8 idx: 0 9 24 45 69
@@ -969,10 +930,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, ThaiResetToTermAfterUtf32) {
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
// Thai
constexpr std::string_view kThai = "ฉันเดินไปทำงานทุกวัน";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- kThai, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ language_segmenter->Segment(kThai));
// String: "ฉันเดินไปทำงานทุกวัน"
// ^ ^ ^ ^ ^ ^
// UTF-8 idx: 0 9 21 27 42 51
@@ -996,10 +955,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, ResetToTermBeforeWordConnectorUtf32) {
auto segmenter, language_segmenter_factory::Create(
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
constexpr std::string_view kText = "package name com:google:android!";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ segmenter->Segment(kText));
// String: "package name com:google:android!"
// ^ ^^ ^^ ^
@@ -1021,10 +978,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, ResetToTermBeforeOutOfBoundsUtf32) {
auto segmenter, language_segmenter_factory::Create(
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
constexpr std::string_view kText = "How are you你好吗お元気ですか";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ segmenter->Segment(kText));
// String: "How are you你好吗お元気ですか"
// ^ ^^ ^^ ^ ^ ^ ^ ^ ^
@@ -1054,15 +1009,13 @@ TEST_P(ReverseJniLanguageSegmenterTest,
constexpr std::string_view kText = "How are𡔖 you你好吗お元気ですか";
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_itr,
- segmenter->Segment(kText,
- LanguageSegmenter::AccessType::kForwardIterator));
+ segmenter->Segment(kText));
std::vector<std::string_view> advance_terms =
GetAllTermsAdvance(advance_itr.get());
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr,
- segmenter->Segment(
- kText, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ segmenter->Segment(kText));
std::vector<std::string_view> reset_terms =
GetAllTermsResetBeforeUtf32(reset_to_term_itr.get());
std::reverse(reset_terms.begin(), reset_terms.end());
@@ -1080,15 +1033,13 @@ TEST_P(ReverseJniLanguageSegmenterTest,
constexpr std::string_view kThai = "ฉันเดินไปทำงานทุกวัน";
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_itr,
- segmenter->Segment(kThai,
- LanguageSegmenter::AccessType::kForwardIterator));
+ segmenter->Segment(kThai));
std::vector<std::string_view> advance_terms =
GetAllTermsAdvance(advance_itr.get());
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr,
- segmenter->Segment(
- kThai, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ segmenter->Segment(kThai));
std::vector<std::string_view> reset_terms =
GetAllTermsResetBeforeUtf32(reset_to_term_itr.get());
std::reverse(reset_terms.begin(), reset_terms.end());
@@ -1105,15 +1056,13 @@ TEST_P(ReverseJniLanguageSegmenterTest,
constexpr std::string_view kKorean = "나는 매일 출근합니다.";
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> advance_itr,
- segmenter->Segment(kKorean,
- LanguageSegmenter::AccessType::kForwardIterator));
+ segmenter->Segment(kKorean));
std::vector<std::string_view> advance_terms =
GetAllTermsAdvance(advance_itr.get());
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> reset_to_term_itr,
- segmenter->Segment(
- kKorean, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ segmenter->Segment(kKorean));
std::vector<std::string_view> reset_terms =
GetAllTermsResetBeforeUtf32(reset_to_term_itr.get());
std::reverse(reset_terms.begin(), reset_terms.end());
@@ -1129,9 +1078,7 @@ TEST_P(ReverseJniLanguageSegmenterTest, MixedLanguagesResetToTermBeforeUtf32) {
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
ICING_ASSERT_OK_AND_ASSIGN(
std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- "How are you你好吗お元気ですか",
- LanguageSegmenter::AccessType::kBidirectionalIterator));
+ language_segmenter->Segment("How are you你好吗お元気ですか"));
// String: "How are you你好吗お元気ですか"
// ^ ^^ ^^ ^ ^ ^ ^ ^ ^
@@ -1169,11 +1116,8 @@ TEST_P(ReverseJniLanguageSegmenterTest,
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
// Multiple continuous whitespaces are treated as one.
constexpr std::string_view kTextWithSpace = "Hello World";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- kTextWithSpace,
- LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ language_segmenter->Segment(kTextWithSpace));
// String: "Hello World"
// ^ ^ ^
@@ -1211,10 +1155,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, ChineseResetToTermBeforeUtf32) {
// CJKT (Chinese, Japanese, Khmer, Thai) are the 4 main languages that
// don't have whitespaces as word delimiter. Chinese
constexpr std::string_view kChinese = "我每天走路去上班。";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- kChinese, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ language_segmenter->Segment(kChinese));
// String: "我每天走路去上班。"
// ^ ^ ^ ^^
// UTF-8 idx: 0 3 9 15 18
@@ -1237,10 +1179,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, JapaneseResetToTermBeforeUtf32) {
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
// Japanese
constexpr std::string_view kJapanese = "私は毎日仕事に歩いています。";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- kJapanese, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ language_segmenter->Segment(kJapanese));
// String: "私は毎日仕事に歩いています。"
// ^ ^ ^ ^ ^ ^ ^ ^ ^
// UTF-8 idx: 0 3 6 12 18212427 33
@@ -1262,10 +1202,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, KhmerResetToTermBeforeUtf32) {
language_segmenter_factory::Create(
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
constexpr std::string_view kKhmer = "ញុំដើរទៅធ្វើការរាល់ថ្ងៃ។";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- kKhmer, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ language_segmenter->Segment(kKhmer));
// String: "ញុំដើរទៅធ្វើការរាល់ថ្ងៃ។"
// ^ ^ ^ ^
// UTF-8 idx: 0 9 24 45
@@ -1288,10 +1226,8 @@ TEST_P(ReverseJniLanguageSegmenterTest, ThaiResetToTermBeforeUtf32) {
GetSegmenterOptions(GetLocale(), jni_cache_.get())));
// Thai
constexpr std::string_view kThai = "ฉันเดินไปทำงานทุกวัน";
- ICING_ASSERT_OK_AND_ASSIGN(
- std::unique_ptr<LanguageSegmenter::Iterator> itr,
- language_segmenter->Segment(
- kThai, LanguageSegmenter::AccessType::kBidirectionalIterator));
+ ICING_ASSERT_OK_AND_ASSIGN(std::unique_ptr<LanguageSegmenter::Iterator> itr,
+ language_segmenter->Segment(kThai));
// String: "ฉันเดินไปทำงานทุกวัน"
// ^ ^ ^ ^ ^ ^
// UTF-8 idx: 0 9 21 27 42 51
generated by cgit v1.2.3 (git 2.25.1) at 2024-06-21 22:58:01 +0000