1 files changed, 41 insertions, 0 deletions
diff --git a/tensorflow_lite_support/ios/text/tokenizers/Sources/TFLTokenizerUtil.mm b/tensorflow_lite_support/ios/text/tokenizers/Sources/TFLTokenizerUtil.mm
new file mode 100644
index 00000000..52180578
--- /dev/null
+++ b/tensorflow_lite_support/ios/text/tokenizers/Sources/TFLTokenizerUtil.mm
@@ -0,0 +1,41 @@
+/* Copyright 2020 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#import "third_party/tensorflow_lite_support/ios/text/tokenizers/Sources/TFLTokenizerUtil.h"
+
+#import "third_party/tensorflow_lite_support/ios/utils/Sources/TFLStringUtil.h"
+
+using ::tflite::support::text::tokenizer::TokenizerResult;
+
+NSArray<NSString *> *Tokenize(Tokenizer *tokenizer, NSString *input) {
+  TokenizerResult tokenize_result = tokenizer->Tokenize(MakeString(input));
+  std::vector<std::string> subwords = tokenize_result.subwords;
+  NSMutableArray<NSString *> *ret = [NSMutableArray arrayWithCapacity:subwords.size()];
+  for (int i = 0; i < subwords.size(); ++i) {
+    [ret addObject:MakeNSString(subwords[i])];
+  }
+  return ret;
+}
+
+NSArray<NSNumber *> *ConvertTokensToIds(Tokenizer *tokenizer, NSArray<NSString *> *tokens) {
+  NSMutableArray<NSNumber *> *ret = [NSMutableArray arrayWithCapacity:[tokens count]];
+  for (NSString *token in tokens) {
+    std::string cc_token = MakeString(token);
+    const char *cToken = cc_token.c_str();
+    int id;
+    tokenizer->LookupId(cToken, &id);
+    [ret addObject:[NSNumber numberWithInt:id]];
+  }
+  return ret;
+}