Merge changes Ie828e013,I25d57384

am: 5457e10513

Change-Id: Icc9843b862be7dfe8de72c90b67cca20a6395b18

4 files changed, 152 insertions, 64 deletions

diff --git a/src/main/java/com/android/vts/servlet/VtsInactivityJobServlet.java b/src/main/java/com/android/vts/servlet/VtsInactivityJobServlet.java
index 9211f15..ee7bc4f 100644
--- a/src/main/java/com/android/vts/servlet/VtsInactivityJobServlet.java
+++ b/src/main/java/com/android/vts/servlet/VtsInactivityJobServlet.java
@@ -21,6 +21,7 @@ import com.android.vts.entity.TestRunEntity;
 import com.android.vts.entity.TestStatusEntity;
 import com.android.vts.util.EmailHelper;
 import com.android.vts.util.FilterUtil;
+import com.android.vts.util.TaskQueueHelper;
 import com.google.appengine.api.datastore.DatastoreService;
 import com.google.appengine.api.datastore.DatastoreServiceFactory;
 import com.google.appengine.api.datastore.Entity;
@@ -145,7 +146,7 @@ public class VtsInactivityJobServlet extends HttpServlet {
                             .method(TaskOptions.Method.POST);
             tasks.add(task);
         }
-        queue.add(tasks);
+        TaskQueueHelper.addToQueue(queue, tasks);
     }
 
     @Override
diff --git a/src/main/java/com/android/vts/servlet/VtsPerformanceJobServlet.java b/src/main/java/com/android/vts/servlet/VtsPerformanceJobServlet.java
index 616f5c7..2b40ed3 100644
--- a/src/main/java/com/android/vts/servlet/VtsPerformanceJobServlet.java
+++ b/src/main/java/com/android/vts/servlet/VtsPerformanceJobServlet.java
@@ -23,27 +23,36 @@ import com.android.vts.util.PerformanceUtil;
 import com.android.vts.util.PerformanceUtil.TimeInterval;
 import com.android.vts.util.ProfilingPointSummary;
 import com.android.vts.util.StatSummary;
+import com.android.vts.util.TaskQueueHelper;
 import com.google.appengine.api.datastore.DatastoreService;
 import com.google.appengine.api.datastore.DatastoreServiceFactory;
 import com.google.appengine.api.datastore.Entity;
 import com.google.appengine.api.datastore.Key;
+import com.google.appengine.api.datastore.KeyFactory;
 import com.google.appengine.api.datastore.Query;
+import com.google.appengine.api.taskqueue.Queue;
+import com.google.appengine.api.taskqueue.QueueFactory;
+import com.google.appengine.api.taskqueue.TaskOptions;
 import java.io.IOException;
 import java.math.RoundingMode;
 import java.text.DecimalFormat;
 import java.text.SimpleDateFormat;
 import java.util.ArrayList;
 import java.util.Date;
-import java.util.HashSet;
 import java.util.List;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 /** Represents the notifications service which is automatically called on a fixed schedule. */
 public class VtsPerformanceJobServlet extends HttpServlet {
+    protected static final Logger logger =
+            Logger.getLogger(VtsPerformanceJobServlet.class.getName());
+
+    private static final String PERFORMANCE_JOB_URL = "/cron/vts_performance_job";
     private static final String MEAN = "Mean";
     private static final String MAX = "Max";
     private static final String MIN = "Min";
@@ -73,9 +82,7 @@ public class VtsPerformanceJobServlet extends HttpServlet {
 
     private static final DecimalFormat FORMATTER;
 
-    /**
-     * Initialize the decimal formatter.
-     */
+    /** Initialize the decimal formatter. */
     static {
         FORMATTER = new DecimalFormat("#.##");
         FORMATTER.setRoundingMode(RoundingMode.HALF_UP);
@@ -83,14 +90,11 @@ public class VtsPerformanceJobServlet extends HttpServlet {
 
     /**
      * Generates an HTML summary of the performance changes for the profiling results in the
-     * specified
-     * table.
+     * specified table.
      *
      * <p>Retrieves the past 24 hours of profiling data and compares it to the 24 hours that
-     * preceded
-     * it. Creates a table representation of the mean and standard deviation for each profiling
-     * point.
-     * When performance degrades, the cell is shaded red.
+     * preceded it. Creates a table representation of the mean and standard deviation for each
+     * profiling point. When performance degrades, the cell is shaded red.
      *
      * @param testName The name of the test whose profiling data to summarize.
      * @param perfSummaries List of PerformanceSummary objects for each profiling run (in reverse
@@ -98,10 +102,9 @@ public class VtsPerformanceJobServlet extends HttpServlet {
      * @param labels List of string labels for use as the column headers.
      * @returns An HTML string containing labeled table summaries.
      */
-    public static String getPeformanceSummary(
+    public static String getPerformanceSummary(
             String testName, List<PerformanceSummary> perfSummaries, List<String> labels) {
-        if (perfSummaries.size() == 0)
-            return "";
+        if (perfSummaries.size() == 0) return "";
         PerformanceSummary now = perfSummaries.get(0);
         String tableHTML = "<p style='" + LABEL_STYLE + "'><b>";
         tableHTML += testName + "</b></p>";
@@ -121,12 +124,9 @@ public class VtsPerformanceJobServlet extends HttpServlet {
             for (int i = 0; i < labels.size(); i++) {
                 String content = labels.get(i);
                 tableHTML += "<th style='" + SECTION_LABEL_STYLE + "' ";
-                if (i == 0)
-                    tableHTML += "colspan='1'";
-                else if (i == 1)
-                    tableHTML += "colspan='3'";
-                else
-                    tableHTML += "colspan='4'";
+                if (i == 0) tableHTML += "colspan='1'";
+                else if (i == 1) tableHTML += "colspan='3'";
+                else tableHTML += "colspan='4'";
                 tableHTML += ">" + content + "</th>";
             }
             tableHTML += "</tr>";
@@ -178,12 +178,18 @@ public class VtsPerformanceJobServlet extends HttpServlet {
                     PerformanceSummary oldPerfSummary = perfSummaries.get(i);
                     if (oldPerfSummary.hasProfilingPoint(profilingPoint)) {
                         StatSummary baseline =
-                                oldPerfSummary.getProfilingPointSummary(profilingPoint)
+                                oldPerfSummary
+                                        .getProfilingPointSummary(profilingPoint)
                                         .getStatSummary(label);
-                        tableHTML += PerformanceUtil.getBestCasePerformanceComparisonHTML(
-                                baseline, stats, "", "", INNER_CELL_STYLE, OUTER_CELL_STYLE);
-                    } else
-                        tableHTML += "<td></td><td></td><td></td><td></td>";
+                        tableHTML +=
+                                PerformanceUtil.getBestCasePerformanceComparisonHTML(
+                                        baseline,
+                                        stats,
+                                        "",
+                                        "",
+                                        INNER_CELL_STYLE,
+                                        OUTER_CELL_STYLE);
+                    } else tableHTML += "<td></td><td></td><td></td><td></td>";
                 }
                 tableHTML += "</tr>";
             }
@@ -196,17 +202,34 @@ public class VtsPerformanceJobServlet extends HttpServlet {
     @Override
     public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
-        Set<Key> allTestKeys = new HashSet<>();
-
+        Queue queue = QueueFactory.getDefaultQueue();
         Query q = new Query(TestEntity.KIND).setKeysOnly();
+        List<TaskOptions> tasks = new ArrayList<>();
         for (Entity test : datastore.prepare(q).asIterable()) {
             if (test.getKey().getName() == null) {
                 continue;
             }
-            allTestKeys.add(test.getKey());
+            TaskOptions task =
+                    TaskOptions.Builder.withUrl(PERFORMANCE_JOB_URL)
+                            .param("testKey", KeyFactory.keyToString(test.getKey()))
+                            .method(TaskOptions.Method.POST);
+            tasks.add(task);
+        }
+        TaskQueueHelper.addToQueue(queue, tasks);
+    }
+
+    @Override
+    public void doPost(HttpServletRequest request, HttpServletResponse response)
+            throws IOException {
+        String testKeyString = request.getParameter("testKey");
+        Key testKey;
+        try {
+            testKey = KeyFactory.stringToKey(testKeyString);
+        } catch (IllegalArgumentException e) {
+            logger.log(Level.WARNING, "Invalid key specified: " + testKeyString);
+            return;
         }
 
-        // Add today to the list of time intervals to analyze
         List<TimeInterval> timeIntervals = new ArrayList<>();
         long nowMilli = System.currentTimeMillis();
         long nowMicro = TimeUnit.MILLISECONDS.toMicros(nowMilli);
@@ -217,10 +240,12 @@ public class VtsPerformanceJobServlet extends HttpServlet {
 
         // Add yesterday as a baseline time interval for analysis
         long oneDayAgo = nowMicro - TimeUnit.DAYS.toMicros(1);
-        String dateStringYesterday = new SimpleDateFormat(
-                "MM-dd-yyyy").format(new Date(TimeUnit.MICROSECONDS.toMillis(oneDayAgo)));
-        TimeInterval yesterday = new TimeInterval(
-                oneDayAgo - TimeUnit.DAYS.toMicros(1), oneDayAgo, dateStringYesterday);
+        String dateStringYesterday =
+                new SimpleDateFormat("MM-dd-yyyy")
+                        .format(new Date(TimeUnit.MICROSECONDS.toMillis(oneDayAgo)));
+        TimeInterval yesterday =
+                new TimeInterval(
+                        oneDayAgo - TimeUnit.DAYS.toMicros(1), oneDayAgo, dateStringYesterday);
         timeIntervals.add(yesterday);
 
         // Add last week as a baseline time interval for analysis
@@ -229,30 +254,28 @@ public class VtsPerformanceJobServlet extends HttpServlet {
         TimeInterval lastWeek = new TimeInterval(oneWeekAgo - oneWeek, oneWeekAgo, LAST_WEEK);
         timeIntervals.add(lastWeek);
 
-        for (Key testKey : allTestKeys) {
-            List<PerformanceSummary> perfSummaries = new ArrayList<>();
-            List<String> labels = new ArrayList<>();
-            labels.add("");
-            for (TimeInterval interval : timeIntervals) {
-                PerformanceSummary perfSummary = new PerformanceSummary();
-                PerformanceUtil.updatePerformanceSummary(
-                        testKey.getName(), interval.start, interval.end, null, perfSummary);
-                if (perfSummary.size() == 0) {
-                    continue;
-                }
-                perfSummaries.add(perfSummary);
-                labels.add(interval.label);
-            }
-            String body = getPeformanceSummary(testKey.getName(), perfSummaries, labels);
-            if (body == null || body.equals("")) {
+        List<PerformanceSummary> perfSummaries = new ArrayList<>();
+        List<String> labels = new ArrayList<>();
+        labels.add("");
+        for (TimeInterval interval : timeIntervals) {
+            PerformanceSummary perfSummary = new PerformanceSummary();
+            PerformanceUtil.updatePerformanceSummary(
+                    testKey.getName(), interval.start, interval.end, null, perfSummary);
+            if (perfSummary.size() == 0) {
                 continue;
             }
-            List<String> emails = EmailHelper.getSubscriberEmails(testKey);
-            if (emails.size() == 0) {
-                continue;
-            }
-            String subject = SUBJECT_PREFIX + testKey.getName();
-            EmailHelper.send(emails, subject, body);
+            perfSummaries.add(perfSummary);
+            labels.add(interval.label);
+        }
+        String body = getPerformanceSummary(testKey.getName(), perfSummaries, labels);
+        if (body == null || body.equals("")) {
+            return;
+        }
+        List<String> emails = EmailHelper.getSubscriberEmails(testKey);
+        if (emails.size() == 0) {
+            return;
         }
+        String subject = SUBJECT_PREFIX + testKey.getName();
+        EmailHelper.send(emails, subject, body);
     }
 }
diff --git a/src/main/java/com/android/vts/util/TaskQueueHelper.java b/src/main/java/com/android/vts/util/TaskQueueHelper.java
new file mode 100644
index 0000000..3d5df53
--- /dev/null
+++ b/src/main/java/com/android/vts/util/TaskQueueHelper.java
@@ -0,0 +1,50 @@
+/**
+ * Copyright 2017 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
+ * except in compliance with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the
+ * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.vts.util;
+
+import com.google.appengine.api.taskqueue.Queue;
+import com.google.appengine.api.taskqueue.TaskOptions;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/** TaskQueueHelper, a helper class for interacting with App Engine Task Queue API. */
+public class TaskQueueHelper {
+    protected static final Logger logger = Logger.getLogger(TaskQueueHelper.class.getName());
+    public static final int MAX_BATCH_ADD_SIZE = 100;
+
+    /**
+     * Add the list of tasks to the provided queue.
+     *
+     * @param queue The task queue in which to insert the tasks.
+     * @param tasks The list of tasks to add.
+     */
+    public static void addToQueue(Queue queue, List<TaskOptions> tasks) {
+        List<TaskOptions> puts = new ArrayList<>();
+        for (TaskOptions task : tasks) {
+            puts.add(task);
+            if (puts.size() == MAX_BATCH_ADD_SIZE) {
+                queue.addAsync(puts);
+                puts = new ArrayList<>();
+            } else if (puts.size() > MAX_BATCH_ADD_SIZE) {
+                logger.log(Level.SEVERE, "Too many tasks batched in the task queue API.");
+                return;
+            }
+        }
+        if (puts.size() > 0) {
+            queue.addAsync(puts);
+        }
+    }
+}
diff --git a/src/test/java/com/android/vts/servlet/VtsPerformanceJobServletTest.java b/src/test/java/com/android/vts/servlet/VtsPerformanceJobServletTest.java
index c8adb69..d15c05b 100644
--- a/src/test/java/com/android/vts/servlet/VtsPerformanceJobServletTest.java
+++ b/src/test/java/com/android/vts/servlet/VtsPerformanceJobServletTest.java
@@ -66,8 +66,16 @@ public class VtsPerformanceJobServletTest {
         for (long value : values) {
             valueList.add(value);
         }
-        return new ProfilingPointRunEntity(KeyFactory.createKey(TestEntity.KIND, "test"), "name", 0,
-                regressionMode.getNumber(), labelList, valueList, "", "", null);
+        return new ProfilingPointRunEntity(
+                KeyFactory.createKey(TestEntity.KIND, "test"),
+                "name",
+                0,
+                regressionMode.getNumber(),
+                labelList,
+                valueList,
+                "",
+                "",
+                null);
     }
 
     /** Asserts whether text is the same as the contents in the baseline file specified. */
@@ -186,11 +194,13 @@ public class VtsPerformanceJobServletTest {
     public void testPerformanceSummaryNormal() throws FileNotFoundException, IOException {
         setUp(false);
         String output =
-                VtsPerformanceJobServlet.getPeformanceSummary("test", dailySummaries, legendLabels);
+                VtsPerformanceJobServlet.getPerformanceSummary(
+                        "test", dailySummaries, legendLabels);
         compareToBaseline(output, "performanceSummary1.html");
     }
 
-    /** End-to-end test of performance report when a profiling point was removed in the latest run.
+    /**
+     * End-to-end test of performance report when a profiling point was removed in the latest run.
      */
     @Test
     public void testPerformanceSummaryDroppedProfilingPoint()
@@ -204,7 +214,8 @@ public class VtsPerformanceJobServletTest {
         summary.update(pt);
         yesterday.insertProfilingPointSummary("p3", summary);
         String output =
-                VtsPerformanceJobServlet.getPeformanceSummary("test", dailySummaries, legendLabels);
+                VtsPerformanceJobServlet.getPerformanceSummary(
+                        "test", dailySummaries, legendLabels);
         compareToBaseline(output, "performanceSummary2.html");
     }
 
@@ -221,17 +232,20 @@ public class VtsPerformanceJobServletTest {
         summary.update(pt);
         today.insertProfilingPointSummary("p3", summary);
         String output =
-                VtsPerformanceJobServlet.getPeformanceSummary("test", dailySummaries, legendLabels);
+                VtsPerformanceJobServlet.getPerformanceSummary(
+                        "test", dailySummaries, legendLabels);
         compareToBaseline(output, "performanceSummary3.html");
     }
 
-    /** End-to-end test of performance report labels are grouped (e.g. as if using unlabeled data)
+    /**
+     * End-to-end test of performance report labels are grouped (e.g. as if using unlabeled data)
      */
     @Test
     public void testPerformanceSummaryGroupedNormal() throws FileNotFoundException, IOException {
         setUp(true);
         String output =
-                VtsPerformanceJobServlet.getPeformanceSummary("test", dailySummaries, legendLabels);
+                VtsPerformanceJobServlet.getPerformanceSummary(
+                        "test", dailySummaries, legendLabels);
         compareToBaseline(output, "performanceSummary4.html");
     }
 }