Snap for 8857176 from d9225b59b17a34d7c0004a7232ad20ef36309e64 to mainline-go-media-releaseaml_go_med_330913000android13-mainline-go-media-release

Change-Id: Ic13f4f2f6cbf41bff9009c63469c94767f015987

5 files changed, 159 insertions, 7 deletions

diff --git a/Android.bp b/Android.bp
index a0fb7f15..2b87f473 100644
--- a/Android.bp
+++ b/Android.bp
@@ -218,7 +218,7 @@ cc_library_shared {
         "tensorflow_lite_support/java/src/native/task/text/nlclassifier/bert_nlclassifier/bert_nl_classifier_jni.cc",
         "tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni.cc",
         "tensorflow_lite_support/java/src/native/task/text/nlclassifier/nl_classifier_jni_utils.cc",
-        "tensorflow_lite_support/java/src/native/task/core/builtin_op_resolver.cc",
+        "tensorflow_lite_support/java/src/native/task/core/minimal_op_resolver.cc",
         "tensorflow_lite_support/cc/utils/jni_utils.cc",
     ],
     shared_libs: ["liblog"],
@@ -265,6 +265,9 @@ java_library {
     srcs: [
         "tensorflow_lite_support/java/src/java/org/tensorflow/lite/task/core/*.java",
     ],
+    optimize: {
+        proguard_flags_files: ["proguard.flags"],
+    },
     apex_available: [
         "//apex_available:platform",
         "com.android.adservices",
diff --git a/proguard.flags b/proguard.flags
new file mode 100644
index 00000000..566e53fd
--- /dev/null
+++ b/proguard.flags
@@ -0,0 +1,10 @@
+# Stop proguard from stripping away code used by tensorflow JNI library.
+-keepclassmembers class org.tensorflow.lite.NativeInterpreterWrapper {
+    private long inferenceDurationNanoseconds;
+}
+
+-keep class org.tensorflow.lite.annotations.UsedByReflection
+-keep @org.tensorflow.lite.annotations.UsedByReflection class *
+-keepclassmembers class * {
+    @org.tensorflow.lite.annotations.UsedByReflection *;
+}
\ No newline at end of file
diff --git a/tensorflow_lite_support/java/src/native/task/core/minimal_op_resolver.cc b/tensorflow_lite_support/java/src/native/task/core/minimal_op_resolver.cc
new file mode 100644
index 00000000..31d693a8
--- /dev/null
+++ b/tensorflow_lite_support/java/src/native/task/core/minimal_op_resolver.cc
@@ -0,0 +1,61 @@
+/* Copyright 2022 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <memory>
+
+#include "absl/memory/memory.h"
+#include "tensorflow/lite/kernels/builtin_op_kernels.h"
+#include "tensorflow/lite/op_resolver.h"
+
+namespace tflite {
+namespace task {
+
+// Create a minimal MutableOpResolver to provide only
+// the ops required by NLClassifier/BertNLClassifier.
+std::unique_ptr<MutableOpResolver> CreateOpResolver() {
+  MutableOpResolver resolver;
+  resolver.AddBuiltin(::tflite::BuiltinOperator_DEQUANTIZE,
+                      ::tflite::ops::builtin::Register_DEQUANTIZE());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_RESHAPE,
+                      ::tflite::ops::builtin::Register_RESHAPE());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_GATHER,
+                      ::tflite::ops::builtin::Register_GATHER());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_STRIDED_SLICE,
+                      ::tflite::ops::builtin::Register_STRIDED_SLICE());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_PAD,
+                      ::tflite::ops::builtin::Register_PAD());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_CONCATENATION,
+                      ::tflite::ops::builtin::Register_CONCATENATION());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_FULLY_CONNECTED,
+                      ::tflite::ops::builtin::Register_FULLY_CONNECTED());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_CAST,
+                      ::tflite::ops::builtin::Register_CAST());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_MUL,
+                      ::tflite::ops::builtin::Register_MUL());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_ADD,
+                      ::tflite::ops::builtin::Register_ADD());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_TRANSPOSE,
+                      ::tflite::ops::builtin::Register_TRANSPOSE());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_SPLIT,
+                      ::tflite::ops::builtin::Register_SPLIT());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_PACK,
+                      ::tflite::ops::builtin::Register_PACK());
+  resolver.AddBuiltin(::tflite::BuiltinOperator_SOFTMAX,
+                      ::tflite::ops::builtin::Register_SOFTMAX());
+  return absl::make_unique<MutableOpResolver>(resolver);
+}
+
+}  // namespace task
+}  // namespace tflite
\ No newline at end of file
diff --git a/tensorflow_lite_support/metadata/cc/metadata_version.cc b/tensorflow_lite_support/metadata/cc/metadata_version.cc
index 7679f6c4..baceca91 100644
--- a/tensorflow_lite_support/metadata/cc/metadata_version.cc
+++ b/tensorflow_lite_support/metadata/cc/metadata_version.cc
@@ -45,6 +45,9 @@ enum class SchemaMembers {
   kSubGraphMetadataInputTensorGroups = 5,
   kSubGraphMetadataOutputTensorGroups = 6,
   kProcessUnitOptionsRegexTokenizerOptions = 7,
+  kContentPropertiesAudioProperties = 8,
+  kAssociatedFileTypeScannIndexFile = 9,
+  kAssociatedFileVersion = 10,
 };
 
 // Helper class to compare semantic versions in terms of three integers, major,
@@ -107,6 +110,12 @@ Version GetMemberVersion(SchemaMembers member) {
       return Version(1, 2, 0);
     case SchemaMembers::kProcessUnitOptionsRegexTokenizerOptions:
       return Version(1, 2, 1);
+    case SchemaMembers::kContentPropertiesAudioProperties:
+      return Version(1, 3, 0);
+    case SchemaMembers::kAssociatedFileTypeScannIndexFile:
+      return Version(1, 4, 0);
+    case SchemaMembers::kAssociatedFileVersion:
+      return Version(1, 4, 1);
     default:
       // Should never happen.
       TFLITE_LOG(FATAL) << "Unsupported schema member: "
@@ -148,6 +157,18 @@ void UpdateMinimumVersionForTable<tflite::AssociatedFile>(
         GetMemberVersion(SchemaMembers::kAssociatedFileTypeVocabulary),
         min_version);
   }
+
+  if (table->type() == AssociatedFileType_SCANN_INDEX_FILE) {
+    UpdateMinimumVersion(
+        GetMemberVersion(SchemaMembers::kAssociatedFileTypeScannIndexFile),
+        min_version);
+  }
+
+  if (table->version() != nullptr) {
+    UpdateMinimumVersion(
+        GetMemberVersion(SchemaMembers::kAssociatedFileVersion),
+        min_version);
+  }
 }
 
 template <>
@@ -177,6 +198,19 @@ void UpdateMinimumVersionForTable<tflite::ProcessUnit>(
 }
 
 template <>
+void UpdateMinimumVersionForTable<tflite::Content>(const tflite::Content* table,
+                                                   Version* min_version) {
+  if (table == nullptr) return;
+
+  // Checks the ContenProperties field.
+  if (table->content_properties_type() == ContentProperties_AudioProperties) {
+    UpdateMinimumVersion(
+        GetMemberVersion(SchemaMembers::kContentPropertiesAudioProperties),
+        min_version);
+  }
+}
+
+template <>
 void UpdateMinimumVersionForTable<tflite::TensorMetadata>(
     const tflite::TensorMetadata* table, Version* min_version) {
   if (table == nullptr) return;
@@ -188,6 +222,9 @@ void UpdateMinimumVersionForTable<tflite::TensorMetadata>(
   // Checks the process_units field.
   UpdateMinimumVersionForArray<tflite::ProcessUnit>(table->process_units(),
                                                     min_version);
+
+  // Check the content field.
+  UpdateMinimumVersionForTable<tflite::Content>(table->content(), min_version);
 }
 
 template <>
diff --git a/tensorflow_lite_support/metadata/metadata_schema.fbs b/tensorflow_lite_support/metadata/metadata_schema.fbs
index 6ce94525..35b3bd94 100644
--- a/tensorflow_lite_support/metadata/metadata_schema.fbs
+++ b/tensorflow_lite_support/metadata/metadata_schema.fbs
@@ -50,7 +50,7 @@ namespace tflite;
 // for which they were added.
 //
 // LINT.IfChange
-// Schema Semantic version: 1.2.1
+// Schema Semantic version: 1.4.1
 // LINT.ThenChange(//tensorflow_lite_support/\
 //     metadata/java/src/java/org/tensorflow/lite/support/metadata/\
 //     MetadataParser.java)
@@ -69,6 +69,9 @@ file_identifier "M001";
 // 1.2.0 - Added input_tensor_group to SubGraphMetadata.
 //         Added output_tensor_group to SubGraphMetadata.
 // 1.2.1 - Added RegexTokenizerOptions to ProcessUnitOptions.
+// 1.3.0 - Added AudioProperties to ContentProperties.
+// 1.4.0 - Added SCANN_INDEX_FILE type to AssociatedFileType.
+// 1.4.1 - Added version to AssociatedFile.
 
 // File extension of any written files.
 file_extension "tflitemeta";
@@ -80,10 +83,12 @@ enum AssociatedFileType : byte {
   // Files such as readme.txt.
   DESCRIPTIONS = 1,
 
-  // Contains labels that annotate certain axis of the tensor. For example,
+  // Contains a list of labels (characters separated by "\n" or in lines) that
+  // annotate certain axis of the tensor. For example,
   // the label file in image classification. Those labels annotate the
   // the output tensor, such that each value in the output tensor is the
-  // probability of that corresponding category specified by the label.
+  // probability of that corresponding category specified by the label. See the
+  // example label file used in image classification [1].
   //
   // <Codegen usage>:
   // If an output tensor has an associated file as TENSOR_AXIS_LABELS, return
@@ -92,12 +97,16 @@ enum AssociatedFileType : byte {
   // If multiple files of the same type are present, the first one is used by
   // default; additional ones are to be distinguished from one another by their
   // specified locale.
+  //
+  // [1]: https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/metadata/python/tests/testdata/image_classifier/labels.txt
   TENSOR_AXIS_LABELS = 2,
 
-  // Contains labels that tensor values correspond to. For example, in
+  // Contains a list of labels (characters separated by "\n" or in lines) that
+  // tensor values correspond to. For example, in
   // the object detection model, one of the output tensors is the detected
   // classes. And each value in the tensor refers to the index of label in the
-  // category label file.
+  // category label file. See the example label file used in object detection
+  // [1].
   //
   // <Codegen usage>:
   // If an output tensor has an associated file as TENSOR_VALUE_LABELS, convert
@@ -105,25 +114,41 @@ enum AssociatedFileType : byte {
   // If multiple files of the same type are present, the first one is used by
   // default; additional ones are to be distinguished from one another by their
   // specified locale.
+  //
+  // [1]: https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/metadata/python/tests/testdata/object_detector/labelmap.txt
   TENSOR_VALUE_LABELS = 3,
 
   // Contains sigmoid-based score calibration parameters, formatted as CSV.
   // Lines contain for each index of an output tensor the scale, slope, offset
   // and (optional) min_score parameters to be used for sigmoid fitting (in this
-  // order and in `strtof`-compatible [1] format).
+  // order and in `strtof`-compatible [1] format). Scale should be a
+  // non-negative value.
   // A line may be left empty to default calibrated scores for this index to
   // default_score.
   // In summary, each line should thus contain 0, 3 or 4 comma-separated values.
   //
+  // See the example score calibration file used in image classification [2].
+  //
   // See documentation for ScoreCalibrationOptions for details.
   //
   // [1]: https://en.cppreference.com/w/c/string/byte/strtof
+  // [2]: https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/metadata/python/tests/testdata/image_classifier/score_calibration.txt
   TENSOR_AXIS_SCORE_CALIBRATION = 4,
 
   // Contains a list of unique words (characters separated by "\n" or in lines)
   // that help to convert natural language words to embedding vectors.
+  //
+  // See the example vocab file used in text classification [1].
+  //
+  // [1]: https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/metadata/python/tests/testdata/nl_classifier/vocab.txt
   // Added in: 1.0.1
   VOCABULARY = 5,
+
+  // TODO(b/222351186): introduce the ScaNN index file with links once the code
+  // is released.
+  // Contains on-devide ScaNN index file with LevelDB format.
+  // Added in: 1.4.0
+  SCANN_INDEX_FILE = 6,
 }
 
 table AssociatedFile {
@@ -154,6 +179,7 @@ table AssociatedFile {
   locale:string;
 
   // Version of the file specified by model creators.
+  // Added in: 1.4.1
   version:string;
 }
 
@@ -235,6 +261,16 @@ enum BoundingBoxType : byte {
 
 }
 
+// The properties for audio tensors.
+// Added in: 1.3.0
+table AudioProperties {
+  // The sample rate in Hz when the audio was captured.
+  sample_rate:uint;
+
+  // The channel count of the audio.
+  channels:uint;
+}
+
 enum CoordinateType : byte {
   // The coordinates are float values from 0 to 1.
   RATIO = 0,
@@ -270,6 +306,8 @@ union ContentProperties {
   FeatureProperties,
   ImageProperties,
   BoundingBoxProperties,
+  // Added in: 1.3.0
+  AudioProperties,
 }
 
 table ValueRange {
@@ -415,6 +453,9 @@ enum ScoreTransformationType : byte {
 // An AssociatedFile with type TANSOR_AXIS_SCORE_CALIBRATION specifying the
 // index-specific parameters must be associated with the corresponding
 // TensorMetadata for score calibration be applied.
+//
+// See the example score calibration file used in image classification [1].
+// [1]: https://github.com/tensorflow/tflite-support/blob/master/tensorflow_lite_support/metadata/python/tests/testdata/image_classifier/score_calibration.txt
 table ScoreCalibrationOptions {
   // The function to use for transforming the uncalibrated score before
   // applying score calibration.