Added json parser for CTS camera performance tests

Test: camera performance test

restore ag/1273728

Change-Id: Ib5c0a67b6bb8f1d2ec06774c54bb8f51ba235cc7

1 files changed, 298 insertions, 80 deletions

diff --git a/src/com/android/media/tests/CameraPerformanceTest.java b/src/com/android/media/tests/CameraPerformanceTest.java
index c24c1b2..ebaad68 100644
--- a/src/com/android/media/tests/CameraPerformanceTest.java
+++ b/src/com/android/media/tests/CameraPerformanceTest.java
@@ -16,19 +16,25 @@
 
 package com.android.media.tests;
 
-import com.google.common.collect.ImmutableMultimap;
-
 import com.android.ddmlib.testrunner.TestIdentifier;
 import com.android.tradefed.config.OptionClass;
 import com.android.tradefed.device.DeviceNotAvailableException;
 import com.android.tradefed.log.LogUtil.CLog;
 import com.android.tradefed.result.ITestInvocationListener;
+import com.android.tradefed.util.FileUtil;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
 
+import org.json.JSONArray;
+import org.json.JSONException;
+import org.json.JSONObject;
 import org.xmlpull.v1.XmlPullParser;
 import org.xmlpull.v1.XmlPullParserException;
 import org.xmlpull.v1.XmlPullParserFactory;
 
 import java.io.ByteArrayInputStream;
+import java.io.File;
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.HashMap;
@@ -38,13 +44,51 @@ import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
 /**
- * This test invocation runs android.hardware.camera2.cts.PerformanceTest -
- * Camera2 API use case performance KPIs, such as camera open time, session creation time,
- * shutter lag etc. The KPI data will be parsed and reported to dashboard.
+ * This test invocation runs android.hardware.camera2.cts.PerformanceTest - Camera2 API use case
+ * performance KPIs (Key Performance Indicator), such as camera open time, session creation time,
+ * shutter lag etc. The KPI data will be parsed and reported.
  */
 @OptionClass(alias = "camera-framework")
 public class CameraPerformanceTest extends CameraTestBase {
 
+    private static final String TEST_CAMERA_LAUNCH = "testCameraLaunch";
+    private static final String TEST_SINGLE_CAPTURE = "testSingleCapture";
+    private static final String TEST_REPROCESSING_LATENCY = "testReprocessingLatency";
+    private static final String TEST_REPROCESSING_THROUGHPUT = "testReprocessingThroughput";
+
+    // KPIs to be reported. The key is test methods and the value is KPIs in the method.
+    private final ImmutableMultimap<String, String> mReportingKpis =
+            new ImmutableMultimap.Builder<String, String>()
+                    .put(TEST_CAMERA_LAUNCH, "Camera launch time")
+                    .put(TEST_CAMERA_LAUNCH, "Camera start preview time")
+                    .put(TEST_SINGLE_CAPTURE, "Camera capture result latency")
+                    .put(TEST_REPROCESSING_LATENCY, "YUV reprocessing shot to shot latency")
+                    .put(TEST_REPROCESSING_LATENCY, "opaque reprocessing shot to shot latency")
+                    .put(TEST_REPROCESSING_THROUGHPUT, "YUV reprocessing capture latency")
+                    .put(TEST_REPROCESSING_THROUGHPUT, "opaque reprocessing capture latency")
+                    .build();
+
+    // JSON format keymap, key is test method name and the value is stream name in Json file
+    private static final ImmutableMap<String, String> METHOD_JSON_KEY_MAP =
+            new ImmutableMap.Builder<String, String>()
+                    .put(TEST_CAMERA_LAUNCH, "test_camera_launch")
+                    .put(TEST_SINGLE_CAPTURE, "test_single_capture")
+                    .put(TEST_REPROCESSING_LATENCY, "test_reprocessing_latency")
+                    .put(TEST_REPROCESSING_THROUGHPUT, "test_reprocessing_throughput")
+                    .build();
+
+    private <E extends Number> double getAverage(List<E> list) {
+        double sum = 0;
+        int size = list.size();
+        for (E num : list) {
+            sum += num.doubleValue();
+        }
+        if (size == 0) {
+            return 0.0;
+        }
+        return (sum / size);
+    }
+
     public CameraPerformanceTest() {
         // Set up the default test info. But this is subject to be overwritten by options passed
         // from commands.
@@ -82,7 +126,9 @@ public class CameraPerformanceTest extends CameraTestBase {
         }
 
         @Override
-        public void handleTestRunEnded(ITestInvocationListener listener, long elapsedTime,
+        public void handleTestRunEnded(
+                ITestInvocationListener listener,
+                long elapsedTime,
                 Map<String, String> runMetrics) {
             // Report metrics at the end of test run.
             Map<String, String> result = parseResult(getAggregatedMetrics());
@@ -91,12 +137,20 @@ public class CameraPerformanceTest extends CameraTestBase {
     }
 
     /**
-     * Parse Camera Performance KPIs result from the stdout generated by each test run.
-     * Then put them all together to post the final report
+     * Parse Camera Performance KPIs results and then put them all together to post the final
+     * report.
      *
      * @return a {@link HashMap} that contains pairs of kpiName and kpiValue
      */
     private Map<String, String> parseResult(Map<String, String> metrics) {
+
+        // if json report exists, return the parse results
+        CtsJsonResultParser ctsJsonResultParser = new CtsJsonResultParser();
+
+        if (ctsJsonResultParser.isJsonFileExist()) {
+            return ctsJsonResultParser.parse();
+        }
+
         Map<String, String> resultsAll = new HashMap<String, String>();
 
         CtsResultParserBase parser;
@@ -122,8 +176,8 @@ public class CameraPerformanceTest extends CameraTestBase {
             Map<String, String> testKpis = parser.parse(testResult, testMethod);
             for (String k : testKpis.keySet()) {
                 if (resultsAll.containsKey(k)) {
-                    throw new RuntimeException(String.format("KPI name (%s) conflicts with " +
-                            "the existing names. ", k));
+                    throw new RuntimeException(
+                            String.format("KPI name (%s) conflicts with the existing names.", k));
                 }
             }
             parser.clear();
@@ -136,31 +190,34 @@ public class CameraPerformanceTest extends CameraTestBase {
 
     public boolean shouldUseCtsXmlResultParser(String result) {
         final String XML_DECLARATION = "<?xml";
-        return (result.startsWith(XML_DECLARATION) ||
-                result.startsWith(XML_DECLARATION.toUpperCase()));
+        return (result.startsWith(XML_DECLARATION)
+                || result.startsWith(XML_DECLARATION.toUpperCase()));
     }
 
-    /**
-     * Data class of CTS test results for Camera framework performance test
-     */
+    /** Data class of CTS test results for Camera framework performance test */
     public static class CtsMetric {
         String testMethod;  // "testSingleCapture"
         String source;      // "android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327"
-                            // or "testSingleCapture" (just test method name)
-        String message;     // "Camera 0: Camera capture latency"
-        String type;        // "lower_better"
+        // or "testSingleCapture" (just test method name)
+        String message; // "Camera 0: Camera capture latency"
+        String type; // "lower_better"
         String unit;        // "ms"
         String value;       // "691.0" (is an average of 736.0 688.0 679.0 667.0 686.0)
         String schemaKey;   // RU schema key = message (+ testMethodName if needed), derived
 
         // eg. "android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327"
-        public static final Pattern SOURCE_REGEX = Pattern.compile(
-                "^(?<package>[a-zA-Z\\d\\._$]+)#(?<method>[a-zA-Z\\d_$]+)(:\\d+)?");
+        public static final Pattern SOURCE_REGEX =
+                Pattern.compile("^(?<package>[a-zA-Z\\d\\._$]+)#(?<method>[a-zA-Z\\d_$]+)(:\\d+)?");
         // eg. "Camera 0: Camera capture latency"
-        public static final Pattern MESSAGE_REGEX = Pattern.compile(
-                "^Camera\\s+(?<cameraId>\\d+):\\s+(?<kpiName>.*)");
-
-        CtsMetric(String testMethod, String source, String message, String type, String unit,
+        public static final Pattern MESSAGE_REGEX =
+                Pattern.compile("^Camera\\s+(?<cameraId>\\d+):\\s+(?<kpiName>.*)");
+
+        CtsMetric(
+                String testMethod,
+                String source,
+                String message,
+                String type,
+                String unit,
                 String value) {
             this.testMethod = testMethod;
             this.source = source;
@@ -181,8 +238,9 @@ public class CameraPerformanceTest extends CameraTestBase {
             // Note 2: Two tests testReprocessingLatency & testReprocessingThroughput have the
             // same metric names to report results. To make the report key name distinct,
             // the test name is added as prefix for these tests for them.
-            final String[] TEST_NAMES_AS_PREFIX = {"testReprocessingLatency",
-                    "testReprocessingThroughput"};
+            final String[] TEST_NAMES_AS_PREFIX = {
+                "testReprocessingLatency", "testReprocessingThroughput"
+            };
             for (String testName : TEST_NAMES_AS_PREFIX) {
                 if (testMethod.endsWith(testName)) {
                     schemaKey = String.format("%s_%s", testName, schemaKey);
@@ -207,17 +265,6 @@ public class CameraPerformanceTest extends CameraTestBase {
      * {@link CtsXmlResultParser} for XML typed format introduced since NYC.
      */
     public abstract class CtsResultParserBase {
-        // KPIs to be reported. The key is test methods and the value is KPIs in the method.
-        private ImmutableMultimap<String, String> mReportingKpis =
-                new ImmutableMultimap.Builder<String, String>()
-                        .put("testCameraLaunch", "Camera launch time")
-                        .put("testCameraLaunch", "Camera start preview time")
-                        .put("testSingleCapture", "Camera capture result latency")
-                        .put("testReprocessingLatency", "YUV reprocessing shot to shot latency")
-                        .put("testReprocessingLatency", "opaque reprocessing shot to shot latency")
-                        .put("testReprocessingThroughput", "YUV reprocessing capture latency")
-                        .put("testReprocessingThroughput", "opaque reprocessing capture latency")
-                        .build();
 
         protected CtsMetric mSummary;
         protected List<CtsMetric> mDetails = new ArrayList<>();
@@ -260,47 +307,32 @@ public class CameraPerformanceTest extends CameraTestBase {
             mSummary = null;
             mDetails.clear();
         }
-
-        public <E extends Number> Double getAverage(List<E> list) {
-            double sum = 0;
-            int size = list.size();
-            for (E num : list) {
-                sum += num.doubleValue();
-            }
-            if (size == 0) {
-                return Double.NaN;
-            }
-            return (sum / size);
-        }
     }
 
     /**
-     * Parses the stdout generated by the underlying instrumentation test
-     * and returns it to test runner for later reporting.
+     * Parses the camera performance test generated by the underlying instrumentation test and
+     * returns it to test runner for later reporting.
      *
-     * Format:
-     *   (summary message)| |(type)|(unit)|(value) ++++
-     *   (source)|(message)|(type)|(unit)|(value)... +++
-     *   ...
+     * <p>TODO(liuyg): Rename this class to not reference CTS.
      *
-     * Example:
-     *   Camera launch average time for Camera 1| |lower_better|ms|586.6++++
-     *   android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171|
-     *       Camera 0: Camera open time|lower_better|ms|74.0 100.0 70.0 67.0 82.0 +++
-     *   android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171|
-     *       Camera 0: Camera configure stream time|lower_better|ms|9.0 5.0 5.0 8.0 5.0
-     *   ...
+     * <p>Format: (summary message)| |(type)|(unit)|(value) ++++
+     * (source)|(message)|(type)|(unit)|(value)... +++ ...
      *
-     * See also com.android.cts.util.ReportLog for the format detail.
+     * <p>Example: Camera launch average time for Camera 1| |lower_better|ms|586.6++++
+     * android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171| Camera 0: Camera open
+     * time|lower_better|ms|74.0 100.0 70.0 67.0 82.0 +++
+     * android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171| Camera 0: Camera configure
+     * stream time|lower_better|ms|9.0 5.0 5.0 8.0 5.0 ...
      *
+     * <p>See also com.android.cts.util.ReportLog for the format detail.
      */
     public class CtsDelimitedResultParser extends CtsResultParserBase {
         private static final String LOG_SEPARATOR = "\\+\\+\\+";
         private static final String SUMMARY_SEPARATOR = "\\+\\+\\+\\+";
-        private Pattern mSummaryRegex =
+        private final Pattern mSummaryRegex =
                 Pattern.compile(
                         "^(?<message>[^|]+)\\| \\|(?<type>[^|]+)\\|(?<unit>[^|]+)\\|(?<value>[0-9 .]+)");
-        private Pattern mDetailRegex =
+        private final Pattern mDetailRegex =
                 Pattern.compile(
                         "^(?<source>[^|]+)\\|(?<message>[^|]+)\\|(?<type>[^|]+)\\|(?<unit>[^|]+)\\|"
                                 + "(?<values>[0-9 .]+)");
@@ -308,6 +340,7 @@ public class CameraPerformanceTest extends CameraTestBase {
         @Override
         public Map<String, String> parse(String result, String testMethod) {
             parseToCtsMetrics(result, testMethod);
+            parseToCtsMetrics(result, testMethod);
             return filter(getDetails(), testMethod);
         }
 
@@ -322,19 +355,22 @@ public class CameraPerformanceTest extends CameraTestBase {
             // Parse summary.
             // Example: "Camera launch average time for Camera 1| |lower_better|ms|586.6++++"
             if (summaryMatcher.matches()) {
-                setSummary(new CtsMetric(testMethod,
-                        null,
-                        summaryMatcher.group("message"),
-                        summaryMatcher.group("type"),
-                        summaryMatcher.group("unit"),
-                        summaryMatcher.group("value")));
+                setSummary(
+                        new CtsMetric(
+                                testMethod,
+                                null,
+                                summaryMatcher.group("message"),
+                                summaryMatcher.group("type"),
+                                summaryMatcher.group("unit"),
+                                summaryMatcher.group("value")));
             } else {
                 // Fall through since the summary is not posted as results.
                 CLog.w("Summary not in the correct format");
             }
 
             // Parse KPIs.
-            // Example: "android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171|Camera 0: Camera open time|lower_better|ms|74.0 100.0 70.0 67.0 82.0 +++"
+            // Example: "android.hardware.camera2.cts.PerformanceTest#testCameraLaunch:171|Camera 0:
+            // Camera open time|lower_better|ms|74.0 100.0 70.0 67.0 82.0 +++"
             String[] details = output[1].split(LOG_SEPARATOR);
             for (String detail : details) {
                 Matcher detailMatcher = mDetailRegex.matcher(detail.trim());
@@ -345,13 +381,14 @@ public class CameraPerformanceTest extends CameraTestBase {
                         values.add(Double.parseDouble(value));
                     }
                     String kpiValue = String.format("%.1f", getAverage(values));
-                    addDetail(new CtsMetric(
-                            testMethod,
-                            detailMatcher.group("source"),
-                            detailMatcher.group("message"),
-                            detailMatcher.group("type"),
-                            detailMatcher.group("unit"),
-                            kpiValue));
+                    addDetail(
+                            new CtsMetric(
+                                    testMethod,
+                                    detailMatcher.group("source"),
+                                    detailMatcher.group("message"),
+                                    detailMatcher.group("type"),
+                                    detailMatcher.group("unit"),
+                                    kpiValue));
                 } else {
                     throw new RuntimeException("KPI not in the correct format");
                 }
@@ -361,7 +398,6 @@ public class CameraPerformanceTest extends CameraTestBase {
 
     /**
      * Parses the CTS test results in a XML format introduced since NYC.
-     *
      * Format:
      *   <Summary>
      *       <Metric source="android.hardware.camera2.cts.PerformanceTest#testSingleCapture:327"
@@ -414,6 +450,7 @@ public class CameraPerformanceTest extends CameraTestBase {
 
         /**
          * Parses a {@link CtsMetric} from the given XML parser.
+         *
          * @param parser
          * @throws IOException
          * @throws XmlPullParserException
@@ -451,4 +488,185 @@ public class CameraPerformanceTest extends CameraTestBase {
             return new CtsMetric(mTestMethod, source, message, type, unit, kpiValue);
         }
     }
+
+    /*
+     * Parse the Json report from the Json String
+     * "test_single_capture":
+     * {"camera_id":"0","camera_capture_latency":[264.0,229.0,229.0,237.0,234.0],
+     * "camera_capture_result_latency":[230.0,197.0,196.0,204.0,202.0]},"
+     * "test_reprocessing_latency":
+     * {"camera_id":"0","format":35,"reprocess_type":"YUV reprocessing",
+     * "capture_message":"shot to shot latency","latency":[102.0,101.0,99.0,99.0,100.0,101.0],
+     * "camera_reprocessing_shot_to_shot_average_latency":100.33333333333333},
+     *
+     * TODO: move this to a seperate class
+     */
+    public class CtsJsonResultParser {
+
+        // report json file set in
+        // cts/tools/cts-tradefed/res/config/cts-preconditions.xml
+        private static final String JSON_RESULT_FILE =
+                "/sdcard/report-log-files/CtsCameraTestCases.reportlog.json";
+        private static final String CAMERA_ID_KEY = "camera_id";
+        private static final String AVERAGE_LATENCY_KEY = "average_latency";
+        private static final String REPROCESS_TYPE_KEY = "reprocess_type";
+        private static final String CAPTURE_MESSAGE_KEY = "capture_message";
+        private static final String LATENCY_KEY = "latency";
+
+        public Map<String, String> parse() {
+
+            Map<String, String> metrics = new HashMap<>();
+
+            String jsonString = getFormatedJsonReportFromFile();
+            if (null == jsonString) {
+                throw new RuntimeException("Get null json report string.");
+            }
+
+            Map<String, List<Double>> metricsData = new HashMap<>();
+
+            try {
+                JSONObject jsonObject = new JSONObject(jsonString);
+
+                for (String testMethod : METHOD_JSON_KEY_MAP.keySet()) {
+
+                    JSONArray jsonArray =
+                            (JSONArray) jsonObject.get(METHOD_JSON_KEY_MAP.get(testMethod));
+
+                    switch (testMethod) {
+                        case TEST_REPROCESSING_THROUGHPUT:
+                        case TEST_REPROCESSING_LATENCY:
+                            for (int i = 0; i < jsonArray.length(); i++) {
+                                JSONObject element = jsonArray.getJSONObject(i);
+
+                                // create a kpiKey from camera id,
+                                // reprocess type and capture message
+                                String cameraId = element.getString(CAMERA_ID_KEY);
+                                String reprocessType = element.getString(REPROCESS_TYPE_KEY);
+                                String captureMessage = element.getString(CAPTURE_MESSAGE_KEY);
+                                String kpiKey =
+                                        String.format(
+                                                "%s_Camera %s %s %s",
+                                                testMethod,
+                                                cameraId,
+                                                reprocessType,
+                                                captureMessage);
+
+                                // read the data array from json object
+                                JSONArray jsonDataArray = element.getJSONArray(LATENCY_KEY);
+                                if (!metricsData.containsKey(kpiKey)) {
+                                    List<Double> list = new ArrayList<>();
+                                    metricsData.put(kpiKey, list);
+                                }
+                                for (int j = 0; j < jsonDataArray.length(); j++) {
+                                    metricsData.get(kpiKey).add(jsonDataArray.getDouble(j));
+                                }
+                            }
+                            break;
+                        case TEST_SINGLE_CAPTURE:
+                        case TEST_CAMERA_LAUNCH:
+                            for (int i = 0; i < jsonArray.length(); i++) {
+                                JSONObject element = jsonArray.getJSONObject(i);
+
+                                String cameraid = element.getString(CAMERA_ID_KEY);
+                                for (String kpiName : mReportingKpis.get(testMethod)) {
+
+                                    // the json key is all lower case
+                                    String jsonKey = kpiName.toLowerCase().replace(" ", "_");
+                                    String kpiKey =
+                                            String.format("Camera %s %s", cameraid, kpiName);
+                                    if (!metricsData.containsKey(kpiKey)) {
+                                        List<Double> list = new ArrayList<>();
+                                        metricsData.put(kpiKey, list);
+                                    }
+                                    JSONArray jsonDataArray = element.getJSONArray(jsonKey);
+                                    for (int j = 0; j < jsonDataArray.length(); j++) {
+                                        metricsData.get(kpiKey).add(jsonDataArray.getDouble(j));
+                                    }
+                                }
+                            }
+                            break;
+                        default:
+                            break;
+                    }
+                }
+            } catch (JSONException e) {
+                CLog.w("JSONException: %s in string %s", e.getMessage(), jsonString);
+            }
+
+            // take the average of all data for reporting
+            for (String kpiKey : metricsData.keySet()) {
+                String kpiValue = String.format("%.1f", getAverage(metricsData.get(kpiKey)));
+                metrics.put(kpiKey, kpiValue);
+            }
+            return metrics;
+        }
+
+        public boolean isJsonFileExist() {
+            try {
+                return getDevice().doesFileExist(JSON_RESULT_FILE);
+            } catch (DeviceNotAvailableException e) {
+                throw new RuntimeException("Failed to check json report file on device.", e);
+            }
+        }
+
+        /*
+         * read json report file on the device
+         */
+        private String getFormatedJsonReportFromFile() {
+            String jsonString = null;
+            try {
+                // pull the json report file from device
+                File outputFile = FileUtil.createTempFile("json", ".txt");
+                getDevice().pullFile(JSON_RESULT_FILE, outputFile);
+                jsonString = reformatJsonString(FileUtil.readStringFromFile(outputFile));
+            } catch (IOException e) {
+                CLog.w("Couldn't parse the output json log file: ", e);
+            } catch (DeviceNotAvailableException e) {
+                CLog.w("Could not pull file: %s, error: %s", JSON_RESULT_FILE, e);
+            }
+            return jsonString;
+        }
+
+        // Reformat the json file to remove duplicate keys
+        private String reformatJsonString(String jsonString) {
+
+            final String TEST_METRICS_PATTERN = "\\\"([a-z0-9_]*)\\\":(\\{[^{}]*\\})";
+            StringBuilder newJsonBuilder = new StringBuilder();
+            // Create map of stream names and json objects.
+            HashMap<String, List<String>> jsonMap = new HashMap<>();
+            Pattern p = Pattern.compile(TEST_METRICS_PATTERN);
+            Matcher m = p.matcher(jsonString);
+            while (m.find()) {
+                String key = m.group(1);
+                String value = m.group(2);
+                if (!jsonMap.containsKey(key)) {
+                    jsonMap.put(key, new ArrayList<String>());
+                }
+                jsonMap.get(key).add(value);
+            }
+            // Rewrite json string as arrays.
+            newJsonBuilder.append("{");
+            boolean firstLine = true;
+            for (String key : jsonMap.keySet()) {
+                if (!firstLine) {
+                    newJsonBuilder.append(",");
+                } else {
+                    firstLine = false;
+                }
+                newJsonBuilder.append("\"").append(key).append("\":[");
+                boolean firstValue = true;
+                for (String stream : jsonMap.get(key)) {
+                    if (!firstValue) {
+                        newJsonBuilder.append(",");
+                    } else {
+                        firstValue = false;
+                    }
+                    newJsonBuilder.append(stream);
+                }
+                newJsonBuilder.append("]");
+            }
+            newJsonBuilder.append("}");
+            return newJsonBuilder.toString();
+        }
+    }
 }