Create servlet for delivering profiling overviews.

Add a servlet for showing profiling statistics for a particular test
plan.

Test: staging
Bug: 38283335
Change-Id: I921313cb43d19a95c53c2a1eaca882b4714ad2e1

4 files changed, 331 insertions, 8 deletions

diff --git a/src/main/java/com/android/vts/servlet/ShowProfilingOverviewServlet.java b/src/main/java/com/android/vts/servlet/ShowProfilingOverviewServlet.java
new file mode 100644
index 0000000..bb1174e
--- /dev/null
+++ b/src/main/java/com/android/vts/servlet/ShowProfilingOverviewServlet.java
@@ -0,0 +1,146 @@
+/*
+ * Copyright (C) 2017 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.android.vts.servlet;
+
+import com.android.vts.entity.ProfilingPointRunEntity;
+import com.android.vts.entity.TestEntity;
+import com.android.vts.entity.TestRunEntity;
+import com.android.vts.proto.VtsReportMessage;
+import com.android.vts.util.BoxPlot;
+import com.android.vts.util.DatastoreHelper;
+import com.android.vts.util.FilterUtil;
+import com.android.vts.util.GraphSerializer;
+import com.android.vts.util.PerformanceUtil;
+import com.google.appengine.api.datastore.DatastoreService;
+import com.google.appengine.api.datastore.DatastoreServiceFactory;
+import com.google.appengine.api.datastore.Entity;
+import com.google.appengine.api.datastore.Key;
+import com.google.appengine.api.datastore.KeyFactory;
+import com.google.appengine.api.datastore.Query;
+import com.google.appengine.api.datastore.Query.Filter;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import javax.servlet.RequestDispatcher;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+/** Servlet for handling requests to load graphs. */
+public class ShowProfilingOverviewServlet extends BaseServlet {
+    private static final String PROFILING_OVERVIEW_JSP = "WEB-INF/jsp/show_profiling_overview.jsp";
+
+    private static final String HIDL_HAL_OPTION = "hidl_hal_mode";
+    private static final String[] splitKeysArray = new String[] {HIDL_HAL_OPTION};
+    private static final Set<String> splitKeySet = new HashSet<>(Arrays.asList(splitKeysArray));
+
+    @Override
+    public PageType getNavParentType() {
+        return PageType.TOT;
+    }
+
+    @Override
+    public List<Page> getBreadcrumbLinks(HttpServletRequest request) {
+        List<Page> links = new ArrayList<>();
+        String testName = request.getParameter("testName");
+        links.add(new Page(PageType.TABLE, testName, "?testName=" + testName));
+
+        links.add(new Page(PageType.GRAPH, "?testName=" + testName));
+        return links;
+    }
+
+    @Override
+    public void doGetHandler(HttpServletRequest request, HttpServletResponse response)
+            throws IOException {
+        RequestDispatcher dispatcher = null;
+        DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
+        String testName = request.getParameter("testName");
+        long endTime = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis());
+        long startTime = endTime - TimeUnit.DAYS.toMicros(14);
+
+        // Create a query for test runs matching the time window filter
+        Key parentKey = KeyFactory.createKey(TestEntity.KIND, testName);
+        Filter profilingFilter =
+                FilterUtil.getProfilingTimeFilter(
+                        parentKey, TestRunEntity.KIND, startTime, endTime);
+        Query profilingQuery =
+                new Query(ProfilingPointRunEntity.KIND)
+                        .setAncestor(parentKey)
+                        .setFilter(profilingFilter);
+        Map<String, BoxPlot> plotMap = new HashMap<>();
+        for (Entity e :
+                datastore
+                        .prepare(profilingQuery)
+                        .asIterable(DatastoreHelper.getLargeBatchOptions())) {
+            ProfilingPointRunEntity pt = ProfilingPointRunEntity.fromEntity(e);
+            if (pt == null
+                    || pt.regressionMode
+                            == VtsReportMessage.VtsProfilingRegressionMode
+                                    .VTS_REGRESSION_MODE_DISABLED) continue;
+            String option = PerformanceUtil.getOptionAlias(pt, splitKeySet);
+
+            if (!plotMap.containsKey(pt.name)) {
+                plotMap.put(pt.name, new BoxPlot(pt.name));
+            }
+
+            BoxPlot plot = plotMap.get(pt.name);
+            long days = (endTime - e.getParent().getId()) / TimeUnit.DAYS.toMicros(1);
+            long time = endTime - days * TimeUnit.DAYS.toMicros(1);
+
+            plot.addSeriesData(Long.toString(time), option, pt);
+        }
+
+        List<BoxPlot> plots = new ArrayList<>();
+        for (String key : plotMap.keySet()) {
+            BoxPlot plot = plotMap.get(key);
+            if (plot.size() == 0) continue;
+            plots.add(plot);
+        }
+        Collections.sort(
+                plots,
+                new Comparator<BoxPlot>() {
+                    @Override
+                    public int compare(BoxPlot b1, BoxPlot b2) {
+                        return b1.getName().compareTo(b2.getName());
+                    }
+                });
+
+        Gson gson =
+                new GsonBuilder()
+                        .registerTypeHierarchyAdapter(BoxPlot.class, new GraphSerializer())
+                        .create();
+        request.setAttribute("plots", gson.toJson(plots));
+        request.setAttribute("testName", request.getParameter("testName"));
+        dispatcher = request.getRequestDispatcher(PROFILING_OVERVIEW_JSP);
+        try {
+            dispatcher.forward(request, response);
+        } catch (ServletException e) {
+            logger.log(Level.SEVERE, "Servlet Exception caught : ", e);
+        }
+    }
+}
diff --git a/src/main/java/com/android/vts/util/BoxPlot.java b/src/main/java/com/android/vts/util/BoxPlot.java
new file mode 100644
index 0000000..b135410
--- /dev/null
+++ b/src/main/java/com/android/vts/util/BoxPlot.java
@@ -0,0 +1,179 @@
+/*
+ * Copyright (c) 2017 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.android.vts.util;
+
+import com.android.vts.entity.ProfilingPointRunEntity;
+import com.google.gson.Gson;
+import com.google.gson.JsonObject;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/** Helper object for describing time-series box plot data. */
+public class BoxPlot extends Graph {
+    private static final String LABEL_KEY = "label";
+    private static final String SERIES_KEY = "seriesList";
+    private static final String MEAN_KEY = "mean";
+    private static final String STD_KEY = "std";
+
+    private final String xLabel = "Day";
+    private String yLabel;
+    private String name;
+    private GraphType type = GraphType.BOX_PLOT;
+    private int count;
+    private final Map<String, ProfilingPointSummary> seriesMap;
+    private final Set<String> labelSet;
+    private final List<String> labels;
+
+    public BoxPlot(String name) {
+        this.name = name;
+        this.count = 0;
+        seriesMap = new HashMap<>();
+        labelSet = new HashSet<>();
+        labels = new ArrayList<>();
+    }
+
+    /**
+     * Get the x axis label.
+     *
+     * @return The x axis label.
+     */
+    @Override
+    public String getXLabel() {
+        return xLabel;
+    }
+
+    /**
+     * Get the graph type.
+     *
+     * @return The graph type.
+     */
+    @Override
+    public GraphType getType() {
+        return type;
+    }
+
+    /**
+     * Get the name of the graph.
+     *
+     * @return The name of the graph.
+     */
+    @Override
+    public String getName() {
+        return name;
+    }
+
+    /**
+     * Get the y axis label.
+     *
+     * @return The y axis label.
+     */
+    @Override
+    public String getYLabel() {
+        return yLabel;
+    }
+
+    /**
+     * Get the number of data points stored in the graph.
+     *
+     * @return The number of data points stored in the graph.
+     */
+    @Override
+    public int size() {
+        return this.count;
+    }
+
+    /**
+     * Add data to the graph.
+     *
+     * @param label The name of the category.
+     * @param profilingPoint The ProfilingPointRunEntity containing data to add.
+     */
+    @Override
+    public void addData(String label, ProfilingPointRunEntity profilingPoint) {
+        addSeriesData(label, "", profilingPoint);
+    }
+
+    /**
+     * Add data to the graph.
+     *
+     * @param label The name of the category.
+     * @param series The data series to add data to.
+     * @param profilingPoint The ProfilingPointRunEntity containing data to add.
+     */
+    public void addSeriesData(String label, String series, ProfilingPointRunEntity profilingPoint) {
+        if (profilingPoint.values.size() == 0)
+            return;
+        if (!seriesMap.containsKey(series)) {
+            seriesMap.put(series, new ProfilingPointSummary());
+        }
+        ProfilingPointSummary summary = seriesMap.get(series);
+        summary.updateLabel(profilingPoint, label);
+        if (labelSet.add(label)) {
+            labels.add(label);
+        }
+        yLabel = profilingPoint.xLabel;
+        ++count;
+    }
+
+    /**
+     * Serializes the graph to json format.
+     *
+     * @return A JsonElement object representing the graph object.
+     */
+    @Override
+    public JsonObject toJson() {
+        JsonObject json = super.toJson();
+        List<JsonObject> stats = new ArrayList<>();
+        List<String> seriesList = new ArrayList<>(seriesMap.keySet());
+        Collections.sort(seriesList);
+        Collections.reverse(labels);
+        for (String label : labels) {
+            JsonObject statJson = new JsonObject();
+            String boxLabel = null;
+            List<JsonObject> statList = new ArrayList<>(seriesList.size());
+            for (String series : seriesList) {
+                ProfilingPointSummary summary = seriesMap.get(series);
+                JsonObject statSummary = new JsonObject();
+                Double mean = null;
+                Double std = null;
+                if (summary.hasLabel(label) && summary.getStatSummary(label).getCount() > 0) {
+                    StatSummary stat = summary.getStatSummary(label);
+                    boxLabel = stat.getLabel();
+                    mean = stat.getMean();
+                    std = 0.;
+                    if (stat.getCount() > 1) {
+                        std = stat.getStd();
+                    }
+                }
+                statSummary.addProperty(MEAN_KEY, mean);
+                statSummary.addProperty(STD_KEY, std);
+                statList.add(statSummary);
+            }
+            statJson.addProperty(LABEL_KEY, boxLabel);
+            statJson.add(VALUE_KEY, new Gson().toJsonTree(statList));
+            stats.add(statJson);
+        }
+        json.add(VALUE_KEY, new Gson().toJsonTree(stats));
+        json.add(SERIES_KEY, new Gson().toJsonTree(seriesList));
+        return json;
+    }
+}
diff --git a/src/main/java/com/android/vts/util/Graph.java b/src/main/java/com/android/vts/util/Graph.java
index 2fe98bb..00c2a3d 100644
--- a/src/main/java/com/android/vts/util/Graph.java
+++ b/src/main/java/com/android/vts/util/Graph.java
@@ -29,7 +29,7 @@ public abstract class Graph {
     public static final String NAME_KEY = "name";
     public static final String TYPE_KEY = "type";
 
-    public static enum GraphType { LINE_GRAPH, HISTOGRAM }
+    public static enum GraphType { LINE_GRAPH, HISTOGRAM, BOX_PLOT }
 
     /**
      * Get the graph type.
diff --git a/src/main/java/com/android/vts/util/ProfilingPointSummary.java b/src/main/java/com/android/vts/util/ProfilingPointSummary.java
index 64a1036..8e0e24a 100644
--- a/src/main/java/com/android/vts/util/ProfilingPointSummary.java
+++ b/src/main/java/com/android/vts/util/ProfilingPointSummary.java
@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016 Google Inc. All Rights Reserved.
+ * Copyright (C) 2017 The Android Open Source Project
  *
  * Licensed under the Apache License, Version 2.0 (the "License"); you
  * may not use this file except in compliance with the License. You may
@@ -96,14 +96,12 @@ public class ProfilingPointSummary implements Iterable<StatSummary> {
     /**
      * Updates the profiling summary at a label with the data from a new profiling report.
      *
-     * <p>Updates the summary specified by the label with all values provided in the report. If
-     * labels
-     * are provided in the report, they will be ignored -- all values are updated only to the
-     * provided
-     * label.
+     * Updates the summary specified by the label with all values provided in the report. If
+     * labels are provided in the report, they will be ignored -- all values are updated only to the
+     * provided label.
      *
      * @param profilingEntity The ProfilingPointRunEntity object containing profiling data.
-     * @param label The ByteString label for which all values in the report will be updated.
+     * @param label The String label for which all values in the report will be updated.
      */
     public void updateLabel(ProfilingPointRunEntity profilingEntity, String label) {
         if (!labelIndices.containsKey(label)) {