Separate test parent and test status information.

Move test status information into a separate entity. This will eliminate
the chance of collision when data is uploaded to the dashboard with the
status job. It will also allow for more parallel processing with other
cron jobs needing state (e.g. coverage).

Test: live on staging
Bug: 62821535
Change-Id: I9c66d6d89a8b6317794f64e918d98e58b5b60b39

6 files changed, 367 insertions, 218 deletions

diff --git a/src/main/java/com/android/vts/entity/TestEntity.java b/src/main/java/com/android/vts/entity/TestEntity.java
index a5664f4..bbba6c6 100644
--- a/src/main/java/com/android/vts/entity/TestEntity.java
+++ b/src/main/java/com/android/vts/entity/TestEntity.java
@@ -16,10 +16,7 @@
 
 package com.android.vts.entity;
 
-import com.android.vts.entity.TestCaseRunEntity.TestCase;
 import com.google.appengine.api.datastore.Entity;
-import java.util.ArrayList;
-import java.util.List;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
@@ -29,91 +26,20 @@ public class TestEntity implements DashboardEntity {
 
     public static final String KIND = "Test";
 
-    // Property keys
-    public static final String PASS_COUNT = "passCount";
-    public static final String FAIL_COUNT = "failCount";
-    public static final String UPDATED_TIMESTAMP = "updatedTimestamp";
-
-    protected static final String FAILING_IDS = "failingTestcaseIds";
-    protected static final String FAILING_OFFSETS = "failingTestcaseOffsets";
-
     public final String testName;
-    public final int passCount;
-    public final int failCount;
-    public final long timestamp;
-    public final List<TestCaseReference> failingTestCases;
-
-    /**
-     * Object representing a reference to a test case.
-     */
-    public static class TestCaseReference {
-        public final long parentId;
-        public final int offset;
-
-        /**
-         * Create a test case reference.
-         * @param parentId The ID of the TestCaseRunEntity containing the test case.
-         * @param offset The offset of the test case into the TestCaseRunEntity.
-         */
-        public TestCaseReference(long parentId, int offset) {
-            this.parentId = parentId;
-            this.offset = offset;
-        }
-
-        /**
-         * Create a test case reference.
-         * @param testCase The TestCase to reference.
-         */
-        public TestCaseReference(TestCase testCase) {
-            this(testCase.parentId, testCase.offset);
-        }
-    }
 
     /**
      * Create a TestEntity object with status metadata.
      *
      * @param testName The name of the test.
-     * @param timestamp The timestamp indicating the most recent test run event in the test state.
-     * @param passCount The number of tests passing up to the timestamp specified.
-     * @param failCount The number of tests failing up to the timestamp specified.
-     * @param failingTestCases The TestCaseReferences of the last observed failing test cases.
-     */
-    public TestEntity(String testName, long timestamp, int passCount, int failCount,
-            List<TestCaseReference> failingTestCases) {
-        this.testName = testName;
-        this.timestamp = timestamp;
-        this.passCount = passCount;
-        this.failCount = failCount;
-        this.failingTestCases = failingTestCases;
-    }
-
-    /**
-     * Create a TestEntity object without metadata.
-     *
-     * @param testName The name of the test.
      */
     public TestEntity(String testName) {
-        this(testName, 0, -1, -1, new ArrayList<TestCaseReference>());
+        this.testName = testName;
     }
 
     @Override
     public Entity toEntity() {
         Entity testEntity = new Entity(KIND, this.testName);
-        if (this.timestamp >= 0 && this.passCount >= 0 && this.failCount >= 0) {
-            testEntity.setProperty(UPDATED_TIMESTAMP, this.timestamp);
-            testEntity.setProperty(PASS_COUNT, this.passCount);
-            testEntity.setProperty(FAIL_COUNT, this.failCount);
-            if (this.failingTestCases.size() > 0) {
-                List<Long> failingTestcaseIds = new ArrayList<>();
-                List<Integer> failingTestcaseOffsets = new ArrayList<>();
-                for (TestCaseReference testCase : this.failingTestCases) {
-                    failingTestcaseIds.add(testCase.parentId);
-                    failingTestcaseOffsets.add(testCase.offset);
-                }
-                testEntity.setUnindexedProperty(FAILING_IDS, failingTestcaseIds);
-                testEntity.setUnindexedProperty(FAILING_OFFSETS, failingTestcaseOffsets);
-            }
-        }
         return testEntity;
     }
 
@@ -130,34 +56,6 @@ public class TestEntity implements DashboardEntity {
             return null;
         }
         String testName = e.getKey().getName();
-        long timestamp = 0;
-        int passCount = -1;
-        int failCount = -1;
-        List<TestCaseReference> failingTestCases = new ArrayList<>();
-        try {
-            if (e.hasProperty(UPDATED_TIMESTAMP)) {
-                timestamp = (long) e.getProperty(UPDATED_TIMESTAMP);
-            }
-            if (e.hasProperty(PASS_COUNT)) {
-                passCount = ((Long) e.getProperty(PASS_COUNT)).intValue();
-            }
-            if (e.hasProperty(FAIL_COUNT)) {
-                failCount = ((Long) e.getProperty(FAIL_COUNT)).intValue();
-            }
-            if (e.hasProperty(FAILING_IDS) && e.hasProperty(FAILING_OFFSETS)) {
-                List<Long> ids = (List<Long>) e.getProperty(FAILING_IDS);
-                List<Long> offsets = (List<Long>) e.getProperty(FAILING_OFFSETS);
-                if (ids.size() == offsets.size()) {
-                    for (int i = 0; i < ids.size(); i++) {
-                        failingTestCases.add(
-                                new TestCaseReference(ids.get(i), offsets.get(i).intValue()));
-                    }
-                }
-            }
-        } catch (ClassCastException exception) {
-            // Invalid contents or null values
-            logger.log(Level.WARNING, "Error parsing test entity.", exception);
-        }
-        return new TestEntity(testName, timestamp, passCount, failCount, failingTestCases);
+        return new TestEntity(testName);
     }
 }
diff --git a/src/main/java/com/android/vts/entity/TestStatusEntity.java b/src/main/java/com/android/vts/entity/TestStatusEntity.java
new file mode 100644
index 0000000..9464352
--- /dev/null
+++ b/src/main/java/com/android/vts/entity/TestStatusEntity.java
@@ -0,0 +1,167 @@
+/*
+ * Copyright (c) 2017 Google Inc. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.android.vts.entity;
+
+import com.android.vts.entity.TestCaseRunEntity.TestCase;
+import com.google.appengine.api.datastore.Entity;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/** Entity describing test status. */
+public class TestStatusEntity implements DashboardEntity {
+    protected static final Logger logger = Logger.getLogger(TestStatusEntity.class.getName());
+
+    public static final String KIND = "TestStatus";
+
+    // Property keys
+    public static final String PASS_COUNT = "passCount";
+    public static final String FAIL_COUNT = "failCount";
+    public static final String UPDATED_TIMESTAMP = "updatedTimestamp";
+
+    protected static final String FAILING_IDS = "failingTestcaseIds";
+    protected static final String FAILING_OFFSETS = "failingTestcaseOffsets";
+
+    public final String testName;
+    public final int passCount;
+    public final int failCount;
+    public final long timestamp;
+    public final List<TestCaseReference> failingTestCases;
+
+    /** Object representing a reference to a test case. */
+    public static class TestCaseReference {
+        public final long parentId;
+        public final int offset;
+
+        /**
+         * Create a test case reference.
+         *
+         * @param parentId The ID of the TestCaseRunEntity containing the test case.
+         * @param offset The offset of the test case into the TestCaseRunEntity.
+         */
+        public TestCaseReference(long parentId, int offset) {
+            this.parentId = parentId;
+            this.offset = offset;
+        }
+
+        /**
+         * Create a test case reference.
+         *
+         * @param testCase The TestCase to reference.
+         */
+        public TestCaseReference(TestCase testCase) {
+            this(testCase.parentId, testCase.offset);
+        }
+    }
+
+    /**
+     * Create a TestEntity object with status metadata.
+     *
+     * @param testName The name of the test.
+     * @param timestamp The timestamp indicating the most recent test run event in the test state.
+     * @param passCount The number of tests passing up to the timestamp specified.
+     * @param failCount The number of tests failing up to the timestamp specified.
+     * @param failingTestCases The TestCaseReferences of the last observed failing test cases.
+     */
+    public TestStatusEntity(
+            String testName,
+            long timestamp,
+            int passCount,
+            int failCount,
+            List<TestCaseReference> failingTestCases) {
+        this.testName = testName;
+        this.timestamp = timestamp;
+        this.passCount = passCount;
+        this.failCount = failCount;
+        this.failingTestCases = failingTestCases;
+    }
+
+    /**
+     * Create a TestEntity object without metadata.
+     *
+     * @param testName The name of the test.
+     */
+    public TestStatusEntity(String testName) {
+        this(testName, 0, -1, -1, new ArrayList<TestCaseReference>());
+    }
+
+    @Override
+    public Entity toEntity() {
+        Entity testEntity = new Entity(KIND, this.testName);
+        if (this.timestamp >= 0 && this.passCount >= 0 && this.failCount >= 0) {
+            testEntity.setProperty(UPDATED_TIMESTAMP, this.timestamp);
+            testEntity.setProperty(PASS_COUNT, this.passCount);
+            testEntity.setProperty(FAIL_COUNT, this.failCount);
+            if (this.failingTestCases.size() > 0) {
+                List<Long> failingTestcaseIds = new ArrayList<>();
+                List<Integer> failingTestcaseOffsets = new ArrayList<>();
+                for (TestCaseReference testCase : this.failingTestCases) {
+                    failingTestcaseIds.add(testCase.parentId);
+                    failingTestcaseOffsets.add(testCase.offset);
+                }
+                testEntity.setUnindexedProperty(FAILING_IDS, failingTestcaseIds);
+                testEntity.setUnindexedProperty(FAILING_OFFSETS, failingTestcaseOffsets);
+            }
+        }
+        return testEntity;
+    }
+
+    /**
+     * Convert an Entity object to a TestEntity.
+     *
+     * @param e The entity to process.
+     * @return TestEntity object with the properties from e processed, or null if incompatible.
+     */
+    @SuppressWarnings("unchecked")
+    public static TestStatusEntity fromEntity(Entity e) {
+        if (!e.getKind().equals(KIND) || e.getKey().getName() == null) {
+            logger.log(Level.WARNING, "Missing test attributes in entity: " + e.toString());
+            return null;
+        }
+        String testName = e.getKey().getName();
+        long timestamp = 0;
+        int passCount = -1;
+        int failCount = -1;
+        List<TestCaseReference> failingTestCases = new ArrayList<>();
+        try {
+            if (e.hasProperty(UPDATED_TIMESTAMP)) {
+                timestamp = (long) e.getProperty(UPDATED_TIMESTAMP);
+            }
+            if (e.hasProperty(PASS_COUNT)) {
+                passCount = ((Long) e.getProperty(PASS_COUNT)).intValue();
+            }
+            if (e.hasProperty(FAIL_COUNT)) {
+                failCount = ((Long) e.getProperty(FAIL_COUNT)).intValue();
+            }
+            if (e.hasProperty(FAILING_IDS) && e.hasProperty(FAILING_OFFSETS)) {
+                List<Long> ids = (List<Long>) e.getProperty(FAILING_IDS);
+                List<Long> offsets = (List<Long>) e.getProperty(FAILING_OFFSETS);
+                if (ids.size() == offsets.size()) {
+                    for (int i = 0; i < ids.size(); i++) {
+                        failingTestCases.add(
+                                new TestCaseReference(ids.get(i), offsets.get(i).intValue()));
+                    }
+                }
+            }
+        } catch (ClassCastException exception) {
+            // Invalid contents or null values
+            logger.log(Level.WARNING, "Error parsing test entity.", exception);
+        }
+        return new TestStatusEntity(testName, timestamp, passCount, failCount, failingTestCases);
+    }
+}
diff --git a/src/main/java/com/android/vts/servlet/DashboardMainServlet.java b/src/main/java/com/android/vts/servlet/DashboardMainServlet.java
index 3766664..a71ac9f 100644
--- a/src/main/java/com/android/vts/servlet/DashboardMainServlet.java
+++ b/src/main/java/com/android/vts/servlet/DashboardMainServlet.java
@@ -16,7 +16,7 @@
 
 package com.android.vts.servlet;
 
-import com.android.vts.entity.TestEntity;
+import com.android.vts.entity.TestStatusEntity;
 import com.android.vts.entity.UserFavoriteEntity;
 import com.google.appengine.api.datastore.DatastoreService;
 import com.google.appengine.api.datastore.DatastoreServiceFactory;
@@ -140,14 +140,17 @@ public class DashboardMainServlet extends BaseServlet {
         String buttonLink;
         String error = null;
 
-        Query q = new Query(TestEntity.KIND)
-                          .addProjection(new PropertyProjection(TestEntity.PASS_COUNT, Long.class))
-                          .addProjection(new PropertyProjection(TestEntity.FAIL_COUNT, Long.class));
+        Query q =
+                new Query(TestStatusEntity.KIND)
+                        .addProjection(
+                                new PropertyProjection(TestStatusEntity.PASS_COUNT, Long.class))
+                        .addProjection(
+                                new PropertyProjection(TestStatusEntity.FAIL_COUNT, Long.class));
         for (Entity test : datastore.prepare(q).asIterable()) {
-            TestEntity testEntity = TestEntity.fromEntity(test);
+            TestStatusEntity status = TestStatusEntity.fromEntity(test);
             if (test != null) {
                 TestDisplay display =
-                        new TestDisplay(test.getKey(), testEntity.passCount, testEntity.failCount);
+                        new TestDisplay(test.getKey(), status.passCount, status.failCount);
                 testMap.put(test.getKey(), display);
                 allTests.add(test.getKey().getName());
             }
@@ -167,8 +170,9 @@ public class DashboardMainServlet extends BaseServlet {
             buttonLink = DASHBOARD_FAVORITES_LINK;
         } else {
             if (testMap.size() > 0) {
-                Filter userFilter = new FilterPredicate(
-                        UserFavoriteEntity.USER, FilterOperator.EQUAL, currentUser);
+                Filter userFilter =
+                        new FilterPredicate(
+                                UserFavoriteEntity.USER, FilterOperator.EQUAL, currentUser);
                 q = new Query(UserFavoriteEntity.KIND).setFilter(userFilter);
 
                 for (Entity favorite : datastore.prepare(q).asIterable()) {
diff --git a/src/main/java/com/android/vts/servlet/VtsAlertJobServlet.java b/src/main/java/com/android/vts/servlet/VtsAlertJobServlet.java
index 9e48108..fe4b14d 100644
--- a/src/main/java/com/android/vts/servlet/VtsAlertJobServlet.java
+++ b/src/main/java/com/android/vts/servlet/VtsAlertJobServlet.java
@@ -20,8 +20,9 @@ import com.android.vts.entity.DeviceInfoEntity;
 import com.android.vts.entity.TestCaseRunEntity;
 import com.android.vts.entity.TestCaseRunEntity.TestCase;
 import com.android.vts.entity.TestEntity;
-import com.android.vts.entity.TestEntity.TestCaseReference;
 import com.android.vts.entity.TestRunEntity;
+import com.android.vts.entity.TestStatusEntity;
+import com.android.vts.entity.TestStatusEntity.TestCaseReference;
 import com.android.vts.proto.VtsReportMessage.TestCaseResult;
 import com.android.vts.util.DatastoreHelper;
 import com.android.vts.util.EmailHelper;
@@ -32,6 +33,7 @@ import com.google.appengine.api.datastore.DatastoreServiceFactory;
 import com.google.appengine.api.datastore.DatastoreTimeoutException;
 import com.google.appengine.api.datastore.Entity;
 import com.google.appengine.api.datastore.EntityNotFoundException;
+import com.google.appengine.api.datastore.FetchOptions;
 import com.google.appengine.api.datastore.Key;
 import com.google.appengine.api.datastore.KeyFactory;
 import com.google.appengine.api.datastore.Query;
@@ -59,11 +61,11 @@ import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import org.apache.commons.lang.StringUtils;
-import org.apache.commons.math3.analysis.function.Add;
 
 /** Represents the notifications service which is automatically called on a fixed schedule. */
 public class VtsAlertJobServlet extends HttpServlet {
     protected final Logger logger = Logger.getLogger(getClass().getName());
+    protected final int MAX_RUN_COUNT = 1000; // maximum number of runs to query for
 
     /**
      * Creates an email footer with the provided link.
@@ -72,36 +74,40 @@ public class VtsAlertJobServlet extends HttpServlet {
      * @return The full HTML email footer.
      */
     private String getFooter(String link) {
-        return "<br><br>For details, visit the"
-                + " <a href='" + link + "'>"
-                + "VTS dashboard.</a>";
+        return "<br><br>For details, visit the" + " <a href='" + link + "'>" + "VTS dashboard.</a>";
     }
 
     /**
      * Compose an email if the test is inactive.
      *
-     * @param test The TestEntity document storing the test status.
+     * @param test The TestStatusEntity document storing the test status.
      * @param link Fully specified link to the test's status page.
      * @param emails The list of email addresses to send the email.
      * @param messages The message list in which to insert the inactivity notification email.
      * @return True if the test is inactive, false otherwise.
      */
     private boolean notifyIfInactive(
-            TestEntity test, String link, List<String> emails, List<Message> messages) {
+            TestStatusEntity test, String link, List<String> emails, List<Message> messages) {
         long now = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis());
         long diff = now - test.timestamp;
         // Send an email daily to notify that the test hasn't been running.
         // After 7 full days have passed, notifications will no longer be sent (i.e. the
         // test is assumed to be deprecated).
-        if (diff > TimeUnit.DAYS.toMicros(1) && diff < TimeUnit.DAYS.toMicros(8)
+        if (diff > TimeUnit.DAYS.toMicros(1)
+                && diff < TimeUnit.DAYS.toMicros(8)
                 && diff % TimeUnit.DAYS.toMicros(1) < TimeUnit.MINUTES.toMicros(3)) {
             Date lastUpload = new Date(TimeUnit.MICROSECONDS.toMillis(test.timestamp));
             String uploadTimeString =
                     new SimpleDateFormat("MM/dd/yyyy HH:mm:ss").format(lastUpload);
             String subject = "Warning! Inactive test: " + test.testName;
-            String body = "Hello,<br><br>Test \"" + test.testName + "\" is inactive. "
-                    + "No new data has been uploaded since " + uploadTimeString + "."
-                    + getFooter(link);
+            String body =
+                    "Hello,<br><br>Test \""
+                            + test.testName
+                            + "\" is inactive. "
+                            + "No new data has been uploaded since "
+                            + uploadTimeString
+                            + "."
+                            + getFooter(link);
             try {
                 messages.add(EmailHelper.composeEmail(emails, subject, body));
                 return true;
@@ -115,18 +121,22 @@ public class VtsAlertJobServlet extends HttpServlet {
     /**
      * Checks whether any new failures have occurred beginning since (and including) startTime.
      *
-     * @param test The TestEntity object for the test.
+     * @param status The TestStatusEntity object for the test.
      * @param link The string URL linking to the test's status table.
-     * @param failedTestCaseMap The map of test case names to TestCase for those failing in the
-     *     last status update.
+     * @param failedTestCaseMap The map of test case names to TestCase for those failing in the last
+     *     status update.
      * @param emailAddresses The list of email addresses to send notifications to.
      * @param messages The email Message queue.
      * @returns latest TestStatusMessage or null if no update is available.
      * @throws IOException
      */
-    public TestEntity getTestStatus(TestEntity test, String link,
-            Map<String, TestCase> failedTestCaseMap, List<String> emailAddresses,
-            List<Message> messages) throws IOException {
+    public TestStatusEntity getTestStatus(
+            TestStatusEntity status,
+            String link,
+            Map<String, TestCase> failedTestCaseMap,
+            List<String> emailAddresses,
+            List<Message> messages)
+            throws IOException {
         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
         String footer = getFooter(link);
 
@@ -141,17 +151,20 @@ public class VtsAlertJobServlet extends HttpServlet {
         Set<String> skippedTestcaseFailures = new HashSet<>();
         Set<String> transientTestcaseFailures = new HashSet<>();
 
-        String testName = test.testName;
+        String testName = status.testName;
         Key testKey = KeyFactory.createKey(TestEntity.KIND, testName);
         Filter testTypeFilter = FilterUtil.getTestTypeFilter(false, true, false);
-        Filter runFilter = FilterUtil.getTimeFilter(
-                testKey, TestRunEntity.KIND, test.timestamp + 1, null, testTypeFilter);
-        Query q = new Query(TestRunEntity.KIND)
-                          .setAncestor(testKey)
-                          .setFilter(runFilter)
-                          .addSort(Entity.KEY_RESERVED_PROPERTY, SortDirection.DESCENDING);
+        Filter runFilter =
+                FilterUtil.getTimeFilter(
+                        testKey, TestRunEntity.KIND, status.timestamp + 1, null, testTypeFilter);
+        Query q =
+                new Query(TestRunEntity.KIND)
+                        .setAncestor(testKey)
+                        .setFilter(runFilter)
+                        .addSort(Entity.KEY_RESERVED_PROPERTY, SortDirection.DESCENDING);
 
-        for (Entity testRun : datastore.prepare(q).asIterable()) {
+        for (Entity testRun :
+                datastore.prepare(q).asIterable(FetchOptions.Builder.withLimit(MAX_RUN_COUNT))) {
             TestRunEntity testRunEntity = TestRunEntity.fromEntity(testRun);
             if (testRunEntity == null) {
                 logger.log(Level.WARNING, "Invalid test run detected: " + testRun.getKey());
@@ -208,7 +221,7 @@ public class VtsAlertJobServlet extends HttpServlet {
         }
 
         if (mostRecentRun == null) {
-            notifyIfInactive(test, link, emailAddresses, messages);
+            notifyIfInactive(status, link, emailAddresses, messages);
             return null;
         }
 
@@ -259,8 +272,7 @@ public class VtsAlertJobServlet extends HttpServlet {
             List<String> sortedNewTestcaseFailures = new ArrayList<>(newTestcaseFailures);
             Collections.sort(sortedNewTestcaseFailures);
             for (String testcaseName : sortedNewTestcaseFailures) {
-                summary += "- "
-                        + "<b>" + testcaseName + "</b><br>";
+                summary += "- " + "<b>" + testcaseName + "</b><br>";
             }
 
             // Add continued test case failures to summary.
@@ -302,8 +314,14 @@ public class VtsAlertJobServlet extends HttpServlet {
 
         if (newTestcaseFailures.size() > 0) {
             String subject = "New test failures in " + testName + " @ " + buildId;
-            String body = "Hello,<br><br>Test cases are failing in " + testName
-                    + " for device build ID(s): " + buildId + ".<br><br>" + summary + footer;
+            String body =
+                    "Hello,<br><br>Test cases are failing in "
+                            + testName
+                            + " for device build ID(s): "
+                            + buildId
+                            + ".<br><br>"
+                            + summary
+                            + footer;
             try {
                 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
             } catch (MessagingException | UnsupportedEncodingException e) {
@@ -311,8 +329,14 @@ public class VtsAlertJobServlet extends HttpServlet {
             }
         } else if (continuedTestcaseFailures.size() > 0) {
             String subject = "Continued test failures in " + testName + " @ " + buildId;
-            String body = "Hello,<br><br>Test cases are failing in " + testName
-                    + " for device build ID(s): " + buildId + ".<br><br>" + summary + footer;
+            String body =
+                    "Hello,<br><br>Test cases are failing in "
+                            + testName
+                            + " for device build ID(s): "
+                            + buildId
+                            + ".<br><br>"
+                            + summary
+                            + footer;
             try {
                 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
             } catch (MessagingException | UnsupportedEncodingException e) {
@@ -320,9 +344,15 @@ public class VtsAlertJobServlet extends HttpServlet {
             }
         } else if (transientTestcaseFailures.size() > 0) {
             String subject = "Transient test failure in " + testName + " @ " + buildId;
-            String body = "Hello,<br><br>Some test cases failed in " + testName + " but tests all "
-                    + "are passing in the latest device build(s): " + buildId + ".<br><br>"
-                    + summary + footer;
+            String body =
+                    "Hello,<br><br>Some test cases failed in "
+                            + testName
+                            + " but tests all "
+                            + "are passing in the latest device build(s): "
+                            + buildId
+                            + ".<br><br>"
+                            + summary
+                            + footer;
             try {
                 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
             } catch (MessagingException | UnsupportedEncodingException e) {
@@ -330,32 +360,42 @@ public class VtsAlertJobServlet extends HttpServlet {
             }
         } else if (fixedTestcases.size() > 0) {
             String subject = "All test cases passing in " + testName + " @ " + buildId;
-            String body = "Hello,<br><br>All test cases passed in " + testName
-                    + " for device build ID(s): " + buildId + "!<br><br>" + summary + footer;
+            String body =
+                    "Hello,<br><br>All test cases passed in "
+                            + testName
+                            + " for device build ID(s): "
+                            + buildId
+                            + "!<br><br>"
+                            + summary
+                            + footer;
             try {
                 messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
             } catch (MessagingException | UnsupportedEncodingException e) {
                 logger.log(Level.WARNING, "Error composing email : ", e);
             }
         }
-        return new TestEntity(test.testName, mostRecentRun.startTimestamp, passingTestcaseCount,
-                failingTestCases.size(), failingTestCases);
+        return new TestStatusEntity(
+                status.testName,
+                mostRecentRun.startTimestamp,
+                passingTestcaseCount,
+                failingTestCases.size(),
+                failingTestCases);
     }
 
     /**
      * Process the current test case failures for a test.
      *
-     * @param testEntity The TestEntity object for the test.
+     * @param status The TestStatusEntity object for the test.
      * @returns a map from test case name to the test case run ID for which the test case failed.
      */
-    public static Map<String, TestCase> getCurrentFailures(TestEntity testEntity) {
-        if (testEntity.failingTestCases == null || testEntity.failingTestCases.size() == 0) {
+    public static Map<String, TestCase> getCurrentFailures(TestStatusEntity status) {
+        if (status.failingTestCases == null || status.failingTestCases.size() == 0) {
             return new HashMap<>();
         }
         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
         Map<String, TestCase> failingTestcases = new HashMap<>();
         Set<Key> gets = new HashSet<>();
-        for (TestCaseReference testCaseRef : testEntity.failingTestCases) {
+        for (TestCaseReference testCaseRef : status.failingTestCases) {
             gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId));
         }
         if (gets.size() == 0) {
@@ -363,7 +403,7 @@ public class VtsAlertJobServlet extends HttpServlet {
         }
         Map<Key, Entity> testCaseMap = datastore.get(gets);
 
-        for (TestCaseReference testCaseRef : testEntity.failingTestCases) {
+        for (TestCaseReference testCaseRef : status.failingTestCases) {
             Key key = KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId);
             if (!testCaseMap.containsKey(key)) {
                 continue;
@@ -382,27 +422,38 @@ public class VtsAlertJobServlet extends HttpServlet {
     @Override
     public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
-        Query q = new Query(TestEntity.KIND);
+        Query q = new Query(TestEntity.KIND).setKeysOnly();
+        List<Key> testKeys = new ArrayList<>();
         for (Entity test : datastore.prepare(q).asIterable()) {
-            TestEntity testEntity = TestEntity.fromEntity(test);
-            if (testEntity == null) {
-                logger.log(Level.WARNING, "Corrupted test entity: " + test.getKey().getName());
-                continue;
+            testKeys.add(test.getKey());
+        }
+        for (Key testKey : testKeys) {
+            String testName = testKey.getName();
+
+            TestStatusEntity status = null;
+            Key statusKey = KeyFactory.createKey(TestStatusEntity.KIND, testName);
+            try {
+                status = TestStatusEntity.fromEntity(datastore.get(statusKey));
+            } catch (EntityNotFoundException e) {
+                // no existing status
+            }
+            if (status == null) {
+                status = new TestStatusEntity(testName);
             }
-            List<String> emails = EmailHelper.getSubscriberEmails(test.getKey());
+            List<String> emails = EmailHelper.getSubscriberEmails(testKey);
 
             StringBuffer fullUrl = request.getRequestURL();
             String baseUrl = fullUrl.substring(0, fullUrl.indexOf(request.getRequestURI()));
-            String link = baseUrl + "/show_table?testName=" + testEntity.testName;
+            String link = baseUrl + "/show_table?testName=" + testName;
 
             List<Message> messageQueue = new ArrayList<>();
-            Map<String, TestCase> failedTestcaseMap = getCurrentFailures(testEntity);
+            Map<String, TestCase> failedTestcaseMap = getCurrentFailures(status);
 
-            TestEntity newTestEntity =
-                    getTestStatus(testEntity, link, failedTestcaseMap, emails, messageQueue);
+            TestStatusEntity newStatus =
+                    getTestStatus(status, link, failedTestcaseMap, emails, messageQueue);
 
             // Send any inactivity notifications
-            if (newTestEntity == null) {
+            if (newStatus == null) {
                 if (messageQueue.size() > 0) {
                     EmailHelper.sendAll(messageQueue);
                 }
@@ -414,26 +465,24 @@ public class VtsAlertJobServlet extends HttpServlet {
                 Transaction txn = datastore.beginTransaction();
                 try {
                     try {
-                        testEntity = TestEntity.fromEntity(datastore.get(test.getKey()));
-
-                        // Another job updated the test entity
-                        if (testEntity == null || testEntity.timestamp >= newTestEntity.timestamp) {
-                            txn.rollback();
-                        } else { // This update is most recent.
-                            datastore.put(newTestEntity.toEntity());
-                            txn.commit();
-                            EmailHelper.sendAll(messageQueue);
-                        }
+                        status = TestStatusEntity.fromEntity(datastore.get(statusKey));
                     } catch (EntityNotFoundException e) {
-                        logger.log(Level.INFO,
-                                "Test disappeared during updated: " + newTestEntity.testName);
+                        // no status left
+                    }
+                    if (status == null || status.timestamp >= newStatus.timestamp) {
+                        txn.rollback();
+                    } else { // This update is most recent.
+                        datastore.put(newStatus.toEntity());
+                        txn.commit();
+                        EmailHelper.sendAll(messageQueue);
                     }
                     break;
-                } catch (ConcurrentModificationException | DatastoreFailureException
+                } catch (ConcurrentModificationException
+                        | DatastoreFailureException
                         | DatastoreTimeoutException e) {
-                    logger.log(Level.WARNING, "Retrying alert job insert: " + test.getKey());
+                    logger.log(Level.WARNING, "Retrying alert job insert: " + statusKey);
                     if (retries++ >= DatastoreHelper.MAX_WRITE_RETRIES) {
-                        logger.log(Level.SEVERE, "Exceeded alert job retries: " + test.getKey());
+                        logger.log(Level.SEVERE, "Exceeded alert job retries: " + statusKey);
                         throw e;
                     }
                 } finally {
diff --git a/src/main/java/com/android/vts/util/DatastoreHelper.java b/src/main/java/com/android/vts/util/DatastoreHelper.java
index 4710b88..6a2856f 100644
--- a/src/main/java/com/android/vts/util/DatastoreHelper.java
+++ b/src/main/java/com/android/vts/util/DatastoreHelper.java
@@ -71,12 +71,12 @@ public class DatastoreHelper {
      * @throws IOException
      */
     public static boolean hasNewer(Key parentKey, String kind, Long lowerBound) throws IOException {
-        if (lowerBound == null || lowerBound <= 0)
-            return false;
+        if (lowerBound == null || lowerBound <= 0) return false;
         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
         Key startKey = KeyFactory.createKey(parentKey, kind, lowerBound);
-        Filter startFilter = new FilterPredicate(
-                Entity.KEY_RESERVED_PROPERTY, FilterOperator.GREATER_THAN, startKey);
+        Filter startFilter =
+                new FilterPredicate(
+                        Entity.KEY_RESERVED_PROPERTY, FilterOperator.GREATER_THAN, startKey);
         Query q = new Query(kind).setAncestor(parentKey).setFilter(startFilter).setKeysOnly();
         return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0;
     }
@@ -91,8 +91,7 @@ public class DatastoreHelper {
      * @throws IOException
      */
     public static boolean hasOlder(Key parentKey, String kind, Long upperBound) throws IOException {
-        if (upperBound == null || upperBound <= 0)
-            return false;
+        if (upperBound == null || upperBound <= 0) return false;
         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
         Key endKey = KeyFactory.createKey(parentKey, kind, upperBound);
         Filter endFilter =
@@ -120,10 +119,11 @@ public class DatastoreHelper {
      */
     public static List<String> getAllProducts() {
         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
-        Query query = new Query(DeviceInfoEntity.KIND)
-                              .addProjection(new PropertyProjection(
-                                      DeviceInfoEntity.PRODUCT, String.class))
-                              .setDistinct(true);
+        Query query =
+                new Query(DeviceInfoEntity.KIND)
+                        .addProjection(
+                                new PropertyProjection(DeviceInfoEntity.PRODUCT, String.class))
+                        .setDistinct(true);
         List<String> devices = new ArrayList<>();
         for (Entity e : datastore.prepare(query).asIterable()) {
             devices.add((String) e.getProperty(DeviceInfoEntity.PRODUCT));
@@ -138,10 +138,11 @@ public class DatastoreHelper {
      */
     public static List<String> getAllBranches() {
         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
-        Query query = new Query(DeviceInfoEntity.KIND)
-                              .addProjection(
-                                      new PropertyProjection(DeviceInfoEntity.BRANCH, String.class))
-                              .setDistinct(true);
+        Query query =
+                new Query(DeviceInfoEntity.KIND)
+                        .addProjection(
+                                new PropertyProjection(DeviceInfoEntity.BRANCH, String.class))
+                        .setDistinct(true);
         List<String> devices = new ArrayList<>();
         for (Entity e : datastore.prepare(query).asIterable()) {
             devices.add((String) e.getProperty(DeviceInfoEntity.BRANCH));
@@ -156,10 +157,11 @@ public class DatastoreHelper {
      */
     public static List<String> getAllBuildFlavors() {
         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
-        Query query = new Query(DeviceInfoEntity.KIND)
-                              .addProjection(new PropertyProjection(
-                                      DeviceInfoEntity.BUILD_FLAVOR, String.class))
-                              .setDistinct(true);
+        Query query =
+                new Query(DeviceInfoEntity.KIND)
+                        .addProjection(
+                                new PropertyProjection(DeviceInfoEntity.BUILD_FLAVOR, String.class))
+                        .setDistinct(true);
         List<String> devices = new ArrayList<>();
         for (Entity e : datastore.prepare(query).asIterable()) {
             devices.add((String) e.getProperty(DeviceInfoEntity.BUILD_FLAVOR));
@@ -176,8 +178,11 @@ public class DatastoreHelper {
         DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
         Set<Entity> puts = new HashSet<>();
 
-        if (!report.hasStartTimestamp() || !report.hasEndTimestamp() || !report.hasTest()
-                || !report.hasHostInfo() || !report.hasBuildInfo()) {
+        if (!report.hasStartTimestamp()
+                || !report.hasEndTimestamp()
+                || !report.hasTest()
+                || !report.hasHostInfo()
+                || !report.hasBuildInfo()) {
             // missing information
             return;
         }
@@ -190,8 +195,9 @@ public class DatastoreHelper {
         Entity testEntity = new TestEntity(testName).toEntity();
         List<Long> testCaseIds = new ArrayList<>();
 
-        Key testRunKey = KeyFactory.createKey(
-                testEntity.getKey(), TestRunEntity.KIND, report.getStartTimestamp());
+        Key testRunKey =
+                KeyFactory.createKey(
+                        testEntity.getKey(), TestRunEntity.KIND, report.getStartTimestamp());
 
         long passCount = 0;
         long failCount = 0;
@@ -306,14 +312,24 @@ public class DatastoreHelper {
         List<String> logLinks = new ArrayList<>();
         // Process log data
         for (LogMessage log : report.getLogList()) {
-            if (!log.hasUrl())
-                continue;
+            if (!log.hasUrl()) continue;
             logLinks.add(log.getUrl().toStringUtf8());
         }
 
-        TestRunEntity testRunEntity = new TestRunEntity(testEntity.getKey(), testRunType,
-                startTimestamp, endTimestamp, testBuildId, hostName, passCount, failCount,
-                testCaseIds, logLinks, coveredLineCount, totalLineCount);
+        TestRunEntity testRunEntity =
+                new TestRunEntity(
+                        testEntity.getKey(),
+                        testRunType,
+                        startTimestamp,
+                        endTimestamp,
+                        testBuildId,
+                        hostName,
+                        passCount,
+                        failCount,
+                        testCaseIds,
+                        logLinks,
+                        coveredLineCount,
+                        totalLineCount);
         puts.add(testRunEntity.toEntity());
 
         int retries = 0;
@@ -329,18 +345,21 @@ public class DatastoreHelper {
                 datastore.put(puts);
                 txn.commit();
                 break;
-            } catch (ConcurrentModificationException | DatastoreFailureException
+            } catch (ConcurrentModificationException
+                    | DatastoreFailureException
                     | DatastoreTimeoutException e) {
                 puts.remove(testEntity);
                 logger.log(Level.WARNING, "Retrying test run insert: " + testEntity.getKey());
                 if (retries++ >= MAX_WRITE_RETRIES) {
-                    logger.log(Level.SEVERE,
+                    logger.log(
+                            Level.SEVERE,
                             "Exceeded maximum test run retries: " + testEntity.getKey());
                     throw e;
                 }
             } finally {
                 if (txn.isActive()) {
-                    logger.log(Level.WARNING,
+                    logger.log(
+                            Level.WARNING,
                             "Transaction rollback forced for run: " + testRunEntity.key);
                     txn.rollback();
                 }
@@ -414,8 +433,17 @@ public class DatastoreHelper {
             logger.log(Level.WARNING, "Couldn't infer test run information from runs.");
             return;
         }
-        TestPlanRunEntity testPlanRun = new TestPlanRunEntity(testPlanEntity.getKey(), testPlanName,
-                type, startTimestamp, endTimestamp, testBuildId, passCount, failCount, testRunKeys);
+        TestPlanRunEntity testPlanRun =
+                new TestPlanRunEntity(
+                        testPlanEntity.getKey(),
+                        testPlanName,
+                        type,
+                        startTimestamp,
+                        endTimestamp,
+                        testBuildId,
+                        passCount,
+                        failCount,
+                        testRunKeys);
 
         // Create the device infos.
         for (DeviceInfoEntity device : devices) {
@@ -436,18 +464,21 @@ public class DatastoreHelper {
                 datastore.put(puts);
                 txn.commit();
                 break;
-            } catch (ConcurrentModificationException | DatastoreFailureException
+            } catch (ConcurrentModificationException
+                    | DatastoreFailureException
                     | DatastoreTimeoutException e) {
                 puts.remove(testPlanEntity);
                 logger.log(Level.WARNING, "Retrying test plan insert: " + testPlanEntity.getKey());
                 if (retries++ >= MAX_WRITE_RETRIES) {
-                    logger.log(Level.SEVERE,
+                    logger.log(
+                            Level.SEVERE,
                             "Exceeded maximum test plan retries: " + testPlanEntity.getKey());
                     throw e;
                 }
             } finally {
                 if (txn.isActive()) {
-                    logger.log(Level.WARNING,
+                    logger.log(
+                            Level.WARNING,
                             "Transaction rollback forced for plan run: " + testPlanRun.key);
                     txn.rollback();
                 }
diff --git a/src/main/webapp/WEB-INF/datastore-indexes.xml b/src/main/webapp/WEB-INF/datastore-indexes.xml
index 01983f5..cef9c54 100644
--- a/src/main/webapp/WEB-INF/datastore-indexes.xml
+++ b/src/main/webapp/WEB-INF/datastore-indexes.xml
@@ -28,7 +28,7 @@
     <property name="__key__" direction="desc"/>
   </datastore-index>
 
-  <datastore-index kind="Test" ancestor="false" source="manual">
+  <datastore-index kind="TestStatus" ancestor="false" source="manual">
     <property name="failCount" direction="asc"/>
     <property name="passCount" direction="asc"/>
   </datastore-index>