tokenization « icing - external/icing.git

Mode	Name	Size
-rw-r--r--	combined-tokenizer_test.cc	11425	log plain
d---------	icu	224	log plain
-rw-r--r--	language-segmenter-factory.h	1663	log plain
-rw-r--r--	language-segmenter-iterator-test-jni-layer.cc	1305	log plain
-rw-r--r--	language-segmenter-iterator_test.cc	8491	log plain
-rw-r--r--	language-segmenter.h	7196	log plain
-rw-r--r--	language-segmenter_benchmark.cc	5155	log plain
-rw-r--r--	plain-tokenizer-test-jni-layer.cc	1236	log plain
-rw-r--r--	plain-tokenizer.cc	5112	log plain
-rw-r--r--	plain-tokenizer.h	1609	log plain
-rw-r--r--	plain-tokenizer_test.cc	23288	log plain
-rw-r--r--	raw-query-tokenizer.cc	27347	log plain
-rw-r--r--	raw-query-tokenizer.h	1620	log plain
-rw-r--r--	raw-query-tokenizer_test.cc	34104	log plain
d---------	reverse_jni	445	log plain
-rw-r--r--	rfc822-tokenizer.cc	28421	log plain
-rw-r--r--	rfc822-tokenizer.h	1169	log plain
-rw-r--r--	rfc822-tokenizer_test.cc	48792	log plain
-rw-r--r--	token.h	2911	log plain
-rw-r--r--	tokenizer-factory.cc	2955	log plain
-rw-r--r--	tokenizer-factory.h	2286	log plain
-rw-r--r--	tokenizer.h	4719	log plain
-rw-r--r--	verbatim-tokenizer.cc	4515	log plain
-rw-r--r--	verbatim-tokenizer.h	1295	log plain
-rw-r--r--	verbatim-tokenizer_test.cc	8991	log plain