Fix logics for floating-point comparision.android-9.0.0_r47android-9.0.0_r46android-9.0.0_r45android-9.0.0_r44android-9.0.0_r43android-9.0.0_r42android-9.0.0_r41android-9.0.0_r40android-9.0.0_r37android-9.0.0_r36pie-qpr3-s1-releasepie-qpr3-releasepie-qpr3-b-releasepie-b4s4-dev

Set the acceptable error range based on both absolute tolerance and
relative tolerance.

Currently, absolute tolerance is set to 1e-5 for FP32 and 5 epsilon
(~5e-3) for FP16 relaxed computation. The relative tolerance is set to
5ULP of the corresponding precision. Add a TODO mark for potential
future adjustment on error limit based on testing.

Bug: 111768023

Test: NeuralNetworksTest_static
Change-Id: I8052d979bc45a83dba4c69ef20b8a740ae640953
Merged-In: I8052d979bc45a83dba4c69ef20b8a740ae640953
(cherry picked from commit 3afdbda56805b6ca1d8ceec66f861c57db616778)

2 files changed, 11 insertions, 5 deletions

diff --git a/nn/runtime/test/TestGenerated.cpp b/nn/runtime/test/TestGenerated.cpp
index c0a5f4cab..d27c184f2 100644
--- a/nn/runtime/test/TestGenerated.cpp
+++ b/nn/runtime/test/TestGenerated.cpp
@@ -82,8 +82,11 @@ static void execute(std::function<void(Model*)> createModel,
     Compilation compilation(&model);
     compilation.finish();
 
-    // If in relaxed mode, set the error range to be 5ULP of FP16.
-    float fpRange = !model.isRelaxed() ? 1e-5f : 5.0f * 0.0009765625f;
+    // TODO: Adjust the error limit based on testing.
+    // If in relaxed mode, set the absolute tolerance to be 5ULP of FP16.
+    float fpAtol = !model.isRelaxed() ? 1e-5f : 5.0f * 0.0009765625f;
+    // Set the relative tolerance to be 5ULP of the corresponding FP precision.
+    float fpRtol = !model.isRelaxed() ? 5.0f * 1.1920928955078125e-7f : 5.0f * 0.0009765625f;
     for (auto& example : examples) {
         SCOPED_TRACE(exampleNo);
         // TODO: We leave it as a copy here.
@@ -123,7 +126,7 @@ static void execute(std::function<void(Model*)> createModel,
         MixedTyped filteredTest = filter(test, isIgnored);
         // We want "close-enough" results for float
 
-        compare(filteredGolden, filteredTest, fpRange);
+        compare(filteredGolden, filteredTest, fpAtol, fpRtol);
         exampleNo++;
     }
 }
diff --git a/nn/tools/test_generator/include/TestHarness.h b/nn/tools/test_generator/include/TestHarness.h
index adbdf8fe0..bb9d21550 100644
--- a/nn/tools/test_generator/include/TestHarness.h
+++ b/nn/tools/test_generator/include/TestHarness.h
@@ -175,9 +175,12 @@ void compare_(
 }
 #undef VALUE_TYPE
 #undef VECTOR_TYPE
-inline void compare(const MixedTyped& golden, const MixedTyped& test, float fpRange = 1e-5f) {
+inline void compare(const MixedTyped& golden, const MixedTyped& test,
+                    float fpAtol = 1e-5f, float fpRtol = 1e-5f) {
     size_t totalNumberOfErrors = 0;
-    compare_<0>(golden, test, [&totalNumberOfErrors, fpRange](float g, float t) {
+    compare_<0>(golden, test, [&totalNumberOfErrors, fpAtol, fpRtol](float g, float t) {
+        // Compute the range based on both absolute tolerance and relative tolerance
+        float fpRange = fpAtol + fpRtol * std::abs(g);
         if (totalNumberOfErrors < gMaximumNumberOfErrorMessages) {
             EXPECT_NEAR(g, t, fpRange);
         }