diff options
Diffstat (limited to 'icing/tokenization/tokenizer.h')
-rw-r--r-- | icing/tokenization/tokenizer.h | 9 |
1 files changed, 0 insertions, 9 deletions
diff --git a/icing/tokenization/tokenizer.h b/icing/tokenization/tokenizer.h index 2bc18cc..24f8269 100644 --- a/icing/tokenization/tokenizer.h +++ b/icing/tokenization/tokenizer.h @@ -40,15 +40,6 @@ class Tokenizer { public: virtual ~Tokenizer() = default; - enum Type { - // Index tokenizers - PLAIN, // Used to tokenize plain text input - VERBATIM, // Used to tokenize the input text in verbatim - - // Query tokenizers - RAW_QUERY, // Used to tokenize raw queries - }; - // An iterator helping to get tokens. // Example usage: // |