Move spec file dumper to TestHarness.

This CL additionally enables the RGG to dump execution results of the
failing drivers, which increases debuggability.

Bug: 150805665
Test: NNT_static_fuzzing
Change-Id: I30ca8b38b9ca626bd06fcad8d233ddbf22bc00d5

5 files changed, 23 insertions, 158 deletions

diff --git a/nn/runtime/test/fuzzing/RandomGraphGenerator.cpp b/nn/runtime/test/fuzzing/RandomGraphGenerator.cpp
index dd516c893..245e0e99b 100644
--- a/nn/runtime/test/fuzzing/RandomGraphGenerator.cpp
+++ b/nn/runtime/test/fuzzing/RandomGraphGenerator.cpp
@@ -273,13 +273,6 @@ TestModel RandomGraph::createTestModel() {
     return testModel;
 }
 
-void RandomGraph::dumpSpecFile(std::string filename, std::string testname = "") {
-    NN_FUZZER_LOG << "Dump Spec File to " << filename;
-    SpecWriter writer(filename, testname);
-    ASSERT_TRUE(writer.isOpen());
-    writer.dump(mOperations, mOperands);
-}
-
 }  // namespace fuzzing_test
 }  // namespace nn
 }  // namespace android
diff --git a/nn/runtime/test/fuzzing/RandomGraphGenerator.h b/nn/runtime/test/fuzzing/RandomGraphGenerator.h
index ffe177ff2..b1391dad6 100644
--- a/nn/runtime/test/fuzzing/RandomGraphGenerator.h
+++ b/nn/runtime/test/fuzzing/RandomGraphGenerator.h
@@ -122,9 +122,6 @@ class RandomGraph {
     // dimensions. The output buffers are only allocated but not initialized.
     test_helper::TestModel createTestModel();
 
-    // Dump the generated random graph to a spec file for debugging and visualization purpose.
-    void dumpSpecFile(std::string filename, std::string testname);
-
     const std::vector<RandomOperation>& getOperations() const { return mOperations; }
 
    private:
diff --git a/nn/runtime/test/fuzzing/RandomGraphGeneratorUtils.cpp b/nn/runtime/test/fuzzing/RandomGraphGeneratorUtils.cpp
index 8251ebc0f..54d468d7e 100644
--- a/nn/runtime/test/fuzzing/RandomGraphGeneratorUtils.cpp
+++ b/nn/runtime/test/fuzzing/RandomGraphGeneratorUtils.cpp
@@ -45,115 +45,6 @@ std::string Logger::getElapsedTime() {
     return oss.str();

 }

 

-SpecWriter::SpecWriter(std::string filename, std::string testname) : os(filename) {

-    if (os.is_open() && testname != "") {

-        os << "# Generated from " << testname << ". Do not edit.\n\n";

-    }

-}

-

-// Main entrance of dumping the spec file.

-// Check nn/tools/test_generator/README.md for guide on spec file syntax.

-void SpecWriter::dump(const std::vector<RandomOperation>& operations,

-                      const std::vector<std::shared_ptr<RandomOperand>>& operands) {

-    // Dump model operands.

-    os << "# Model operands\n";

-    for (auto& operand : operands) dump(operand);

-

-    // Dump model operations.

-    os << "\n# Model operations\nmodel = Model()\n";

-    for (const auto& operation : operations) dump(operation);

-

-    // Dump input/output buffers.

-    os << "\n# Example\n";

-    os << "Example({\n";

-    for (auto& operand : operands) {

-        if (operand->type == RandomOperandType::CONST ||

-            operand->type == RandomOperandType::INTERNAL) {

-            continue;

-        }

-        os << "    op" << operand->opIndex << ": [";

-        dump(operand->dataType, reinterpret_cast<uint8_t*>(operand->buffer.data()),

-             operand->getNumberOfElements());

-        os << "],\n";

-    }

-    os << "}).DisableDynamicOutputShapeVariation().DisableLifeTimeVariation()\n";

-}

-

-// Dump an operand buffer.

-void SpecWriter::dump(test_wrapper::Type type, const uint8_t* buffer, uint32_t length) {

-    if (buffer == nullptr) return;

-    switch (type) {

-        case test_wrapper::Type::FLOAT32:

-        case test_wrapper::Type::TENSOR_FLOAT32:

-            dump(reinterpret_cast<const float*>(buffer), length);

-            break;

-        case test_wrapper::Type::INT32:

-        case test_wrapper::Type::TENSOR_INT32:

-            dump(reinterpret_cast<const int32_t*>(buffer), length);

-            break;

-        case test_wrapper::Type::TENSOR_QUANT8_ASYMM:

-            dump(reinterpret_cast<const uint8_t*>(buffer), length);

-            break;

-        case test_wrapper::Type::TENSOR_QUANT8_SYMM:

-            dump(reinterpret_cast<const int8_t*>(buffer), length);

-            break;

-        case test_wrapper::Type::TENSOR_QUANT16_ASYMM:

-            dump(reinterpret_cast<const uint16_t*>(buffer), length);

-            break;

-        case test_wrapper::Type::TENSOR_QUANT16_SYMM:

-            dump(reinterpret_cast<const int16_t*>(buffer), length);

-            break;

-        case test_wrapper::Type::BOOL:

-        case test_wrapper::Type::TENSOR_BOOL8:

-            dump(reinterpret_cast<const bool8*>(buffer), length);

-            break;

-        case test_wrapper::Type::FLOAT16:

-        case test_wrapper::Type::TENSOR_FLOAT16:

-            dump(reinterpret_cast<const _Float16*>(buffer), length);

-            break;

-        default:

-            NN_FUZZER_CHECK(false) << "Unknown type when dumping the buffer";

-    }

-}

-

-// Dump dimensions with curly braces.

-void SpecWriter::dump(const std::vector<RandomVariable>& dimensions) {

-    os << "{" << joinStr(", ", dimensions, [](const RandomVariable& var) {

-        return std::to_string(var.getValue());

-    }) << "}";

-}

-

-// Dump a model operand.

-// e.g. op0 = Input("op0", "TENSOR_FLOAT32", "{1, 2, 6, 1}")

-// e.g. op1 = Parameter("op1", "INT32", "{}", [2])

-void SpecWriter::dump(const std::shared_ptr<RandomOperand>& op) {

-    os << "op" << op->opIndex << " = " << toString(op->type) << "(\"op" << op->opIndex << "\", \""

-       << toString(op->dataType) << "\", \"";

-    dump(op->dimensions);

-    if (op->scale != 0.0f || op->zeroPoint != 0) {

-        os << ", " << op->scale << "f, " << op->zeroPoint;

-    }

-    os << "\"";

-    if (op->type == RandomOperandType::CONST) {

-        os << ", [";

-        dump(op->dataType, reinterpret_cast<uint8_t*>(op->buffer.data()),

-             op->getNumberOfElements());

-        os << "]";

-    }

-    os << ")\n";

-}

-

-// Dump a model operation.

-// e.g. model = model.Operation("CONV_2D", op0, op1, op2, op3, op4, op5, op6).To(op7)

-void SpecWriter::dump(const RandomOperation& op) {

-    auto getOperandStr = [](std::shared_ptr<RandomOperand> op) {

-        return "op" + std::to_string(op->opIndex);

-    };

-    os << "model = model.Operation(\"" << kOperationNames[op.opType] << "\", "

-       << joinStr(", ", op.inputs, getOperandStr) << ").To("

-       << joinStr(", ", op.outputs, getOperandStr) << ")\n";

-}

-

 }  // namespace fuzzing_test

 }  // namespace nn

 }  // namespace android

diff --git a/nn/runtime/test/fuzzing/RandomGraphGeneratorUtils.h b/nn/runtime/test/fuzzing/RandomGraphGeneratorUtils.h
index 80987d598..ddf1a84d8 100644
--- a/nn/runtime/test/fuzzing/RandomGraphGeneratorUtils.h
+++ b/nn/runtime/test/fuzzing/RandomGraphGeneratorUtils.h
@@ -375,39 +375,6 @@ class bool8 {
 };
 static_assert(sizeof(bool8) == 1, "size of bool8 must be 8 bits");
 
-// Dump the random graph to a spec file.
-class SpecWriter {
-   public:
-    SpecWriter(std::string filename, std::string testname = "");
-    bool isOpen() { return os.is_open(); }
-    void dump(const std::vector<RandomOperation>& operations,
-              const std::vector<std::shared_ptr<RandomOperand>>& operands);
-
-   private:
-    void dump(test_wrapper::Type type, const uint8_t* buffer, uint32_t length);
-    void dump(const std::vector<RandomVariable>& dimensions);
-    void dump(const std::shared_ptr<RandomOperand>& op);
-    void dump(const RandomOperation& op);
-
-    template <typename T>
-    void dump(const T* buffer, uint32_t length) {
-        for (uint32_t i = 0; i < length; i++) {
-            if (i != 0) os << ", ";
-            if constexpr (std::is_integral<T>::value) {
-                os << static_cast<int>(buffer[i]);
-            } else if constexpr (std::is_same<T, _Float16>::value) {
-                os << static_cast<float>(buffer[i]);
-            } else if constexpr (std::is_same<T, bool8>::value) {
-                os << (buffer[i] ? "True" : "False");
-            } else {
-                os << buffer[i];
-            }
-        }
-    }
-
-    std::ofstream os;
-};
-
 struct RandomNumberGenerator {
     static std::mt19937 generator;
 };
diff --git a/nn/runtime/test/fuzzing/TestRandomGraph.cpp b/nn/runtime/test/fuzzing/TestRandomGraph.cpp
index 32ebd7560..ffc586474 100644
--- a/nn/runtime/test/fuzzing/TestRandomGraph.cpp
+++ b/nn/runtime/test/fuzzing/TestRandomGraph.cpp
@@ -180,9 +180,6 @@ class RandomGraphTest : public ::testing::TestWithParam<uint32_t> {
     }
 
     virtual void TearDown() override {
-        if (::testing::Test::HasFailure() || mDumpSpec) {
-            mGraph.dumpSpecFile("/data/local/tmp/" + mTestName + ".mod.py", mTestName);
-        }
         NN_FUZZER_LOG_CLOSE;
 #ifndef NNTEST_CTS
         if (mDetectMemoryLeak) {
@@ -283,6 +280,10 @@ class RandomGraphTest : public ::testing::TestWithParam<uint32_t> {
         if (featureLevel >= __ANDROID_API_Q__ && !isRef) {
             checkResults(mTestModel, outputs, mCriteria);
         }
+
+        if (::testing::Test::HasFailure() || mDumpSpec) {
+            dumpTestResults(name, outputs);
+        }
     }
 
     // Compile and execute the generated graph normally (i.e., allow runtime to
@@ -311,10 +312,11 @@ class RandomGraphTest : public ::testing::TestWithParam<uint32_t> {
     // Main test entrance.
     void testRandomGraph(uint32_t numOperations, uint32_t dimensionRange) {
         // Generate a random graph.
-        ASSERT_TRUE(mGraph.generate(kSeed, numOperations, dimensionRange));
+        RandomGraph graph;
+        ASSERT_TRUE(graph.generate(kSeed, numOperations, dimensionRange));
 
         // Create a model from the random graph.
-        mTestModel = mGraph.createTestModel();
+        mTestModel = graph.createTestModel();
 
         generated_tests::GeneratedModel model;
         generated_tests::createModel(mTestModel, &model);
@@ -356,6 +358,19 @@ class RandomGraphTest : public ::testing::TestWithParam<uint32_t> {
         }
     }
 
+    void dumpTestResults(const std::string& deviceName, const std::vector<TestBuffer>& results) {
+        // The dumper is constructed lazily -- only create the file and dump the test model at the
+        // first time this function is called.
+        if (mDumper == nullptr) {
+            mOutStream.open("/data/local/tmp/" + mTestName + ".mod.py");
+            ASSERT_TRUE(mOutStream.is_open());
+            mOutStream << "# Generated from " << mTestName << ". Do not edit.\n\n";
+            mDumper = std::make_unique<SpecDumper>(mTestModel, mOutStream);
+            mDumper->dumpTestModel();
+        }
+        mDumper->dumpResults(deviceName, results);
+    }
+
     enum GraphSize : uint32_t { SINGLE = 1, SMALL = 5, LARGE = 40 };
     enum DimensionRange : uint32_t { NARROW = 10, WIDE = 1000 };
 
@@ -366,10 +381,12 @@ class RandomGraphTest : public ::testing::TestWithParam<uint32_t> {
 
     const uint32_t kSeed = GetParam();
     std::string mTestName;
-    RandomGraph mGraph;
     TestModel mTestModel;
     AccuracyCriteria mCriteria;
 
+    std::ofstream mOutStream;
+    std::unique_ptr<SpecDumper> mDumper;
+
     static int64_t mStandardDevicesFeatureLevel;  // minimum across all devices
 #ifndef NNTEST_CTS
     static std::vector<std::shared_ptr<Device>> mStandardDevices;