tokenization « icing - external/icing.git

Mode	Name	Size
-rw-r--r--	combined-tokenizer_test.cc	10037	log plain
d---------	icu	224	log plain
-rw-r--r--	language-segmenter-factory.h	1664	log plain
-rw-r--r--	language-segmenter-iterator-test-jni-layer.cc	1305	log plain
-rw-r--r--	language-segmenter-iterator_test.cc	8491	log plain
-rw-r--r--	language-segmenter.h	7196	log plain
-rw-r--r--	language-segmenter_benchmark.cc	5143	log plain
-rw-r--r--	plain-tokenizer-test-jni-layer.cc	1236	log plain
-rw-r--r--	plain-tokenizer.cc	4945	log plain
-rw-r--r--	plain-tokenizer.h	1609	log plain
-rw-r--r--	plain-tokenizer_test.cc	21210	log plain
-rw-r--r--	raw-query-tokenizer.cc	27296	log plain
-rw-r--r--	raw-query-tokenizer.h	1620	log plain
-rw-r--r--	raw-query-tokenizer_test.cc	32784	log plain
d---------	reverse_jni	445	log plain
-rw-r--r--	token.h	1764	log plain
-rw-r--r--	tokenizer-factory.cc	2440	log plain
-rw-r--r--	tokenizer-factory.h	2286	log plain
-rw-r--r--	tokenizer.h	4641	log plain
-rw-r--r--	verbatim-tokenizer.cc	4389	log plain
-rw-r--r--	verbatim-tokenizer.h	1295	log plain
-rw-r--r--	verbatim-tokenizer_test.cc	8896	log plain