Updating datastore API for HalApiEntity

Test: go/vts-web-staging
Bug: 117130291
Change-Id: Iae1fa2f92e7f0ddbb7e1b4fa4ff0c3d41ed3ae70

5 files changed, 518 insertions, 83 deletions

diff --git a/src/main/java/com/android/vts/api/BaseApiServlet.java b/src/main/java/com/android/vts/api/BaseApiServlet.java
index 1ad0237..b7384a7 100644
--- a/src/main/java/com/android/vts/api/BaseApiServlet.java
+++ b/src/main/java/com/android/vts/api/BaseApiServlet.java
@@ -17,61 +17,51 @@
 package com.android.vts.api;
 
 import com.google.apphosting.api.ApiProxy;
-import com.google.gson.Gson;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
 import java.util.Properties;
 import java.util.logging.Logger;
 import javax.servlet.ServletConfig;
 import javax.servlet.ServletException;
 import javax.servlet.http.HttpServlet;
-import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
-/**
- * An abstract class to be subclassed to create API Servlet
- */
+/** An abstract class to be subclassed to create API Servlet */
 public class BaseApiServlet extends HttpServlet {
 
-  private static final Logger logger =
-      Logger.getLogger(BaseApiServlet.class.getName());
+    private static final Logger logger = Logger.getLogger(BaseApiServlet.class.getName());
+
+    /** System Configuration Property class */
+    protected Properties systemConfigProp = new Properties();
 
-  /**
-   * System Configuration Property class
-   */
-  protected Properties systemConfigProp = new Properties();
+    /** Appengine server host name */
+    protected String hostName;
 
-  /**
-   * Appengine server host name
-   */
-  protected String hostName;
+    /**
+     * This variable is for maximum number of entities per transaction You can find the detail here
+     * (https://cloud.google.com/datastore/docs/concepts/limits)
+     */
+    protected int MAX_ENTITY_SIZE_PER_TRANSACTION = 300;
 
-  @Override
-  public void init(ServletConfig cfg) throws ServletException {
-    super.init(cfg);
+    @Override
+    public void init(ServletConfig cfg) throws ServletException {
+        super.init(cfg);
 
-    ApiProxy.Environment env = ApiProxy.getCurrentEnvironment();
-    hostName = env.getAttributes().get("com.google.appengine.runtime.default_version_hostname")
-        .toString();
-    try {
-      InputStream defaultInputStream =
-          BaseApiServlet.class
-              .getClassLoader()
-              .getResourceAsStream("config.properties");
-      systemConfigProp.load(defaultInputStream);
+        systemConfigProp =
+                Properties.class.cast(cfg.getServletContext().getAttribute("systemConfigProp"));
 
-    } catch (FileNotFoundException e) {
-      e.printStackTrace();
-    } catch (IOException e) {
-      e.printStackTrace();
+        this.MAX_ENTITY_SIZE_PER_TRANSACTION =
+                Integer.parseInt(systemConfigProp.getProperty("datastore.maxEntitySize"));
+
+        ApiProxy.Environment env = ApiProxy.getCurrentEnvironment();
+        hostName =
+                env.getAttributes()
+                        .get("com.google.appengine.runtime.default_version_hostname")
+                        .toString();
     }
-  }
 
-  protected void setAccessControlHeaders(HttpServletResponse resp) {
-    resp.setHeader("Access-Control-Allow-Origin", hostName);
-    resp.setHeader("Access-Control-Allow-Methods", "GET, PUT, POST, OPTIONS, DELETE");
-    resp.addHeader("Access-Control-Allow-Headers", "Content-Type");
-    resp.addHeader("Access-Control-Max-Age", "86400");
-  }
+    protected void setAccessControlHeaders(HttpServletResponse resp) {
+        resp.setHeader("Access-Control-Allow-Origin", hostName);
+        resp.setHeader("Access-Control-Allow-Methods", "GET, PUT, POST, OPTIONS, DELETE");
+        resp.addHeader("Access-Control-Allow-Headers", "Content-Type");
+        resp.addHeader("Access-Control-Max-Age", "86400");
+    }
 }
diff --git a/src/main/java/com/android/vts/api/BigtableLegacyJsonServlet.java b/src/main/java/com/android/vts/api/BigtableLegacyJsonServlet.java
index ac69b84..d331d4b 100644
--- a/src/main/java/com/android/vts/api/BigtableLegacyJsonServlet.java
+++ b/src/main/java/com/android/vts/api/BigtableLegacyJsonServlet.java
@@ -150,7 +150,6 @@ public class BigtableLegacyJsonServlet extends HttpServlet {
         try {
             byte[] value = Base64.decodeBase64(payloadJson.getString("value"));
             TestReportMessage testReportMessage = TestReportMessage.parseFrom(value);
-            DatastoreHelper.insertTestReport(testReportMessage);
         } catch (InvalidProtocolBufferException e) {
             logger.log(Level.WARNING, "Invalid report posted to dashboard.");
         }
diff --git a/src/main/java/com/android/vts/api/CoverageRestServlet.java b/src/main/java/com/android/vts/api/CoverageRestServlet.java
index 217b740..165db8d 100644
--- a/src/main/java/com/android/vts/api/CoverageRestServlet.java
+++ b/src/main/java/com/android/vts/api/CoverageRestServlet.java
@@ -20,8 +20,10 @@ import com.android.vts.entity.ApiCoverageEntity;
 import com.android.vts.entity.CoverageEntity;
 import com.android.vts.entity.TestCoverageStatusEntity;
 import com.android.vts.entity.TestPlanRunEntity;
+import com.android.vts.entity.TestRunEntity;
 import com.google.gson.Gson;
 import com.googlecode.objectify.Key;
+
 import java.io.IOException;
 import java.util.ArrayList;
 import java.util.Collection;
@@ -104,9 +106,11 @@ public class CoverageRestServlet extends BaseApiServlet {
         List<List<String>> allCoveredHalApiList = new ArrayList();
 
         Key<TestPlanRunEntity> key = Key.create(urlSafeKey);
-        TestPlanRunEntity testPlanRunEntity = ofy().load().key(key).now();
+        System.out.println("urlSafekey => " + urlSafeKey);
+        TestPlanRunEntity testPlanRunEntity = ofy().load().key(key).safe();
 
-        for (Key testRunKey : testPlanRunEntity.getTestRuns()) {
+        System.out.println("testPlanRunEntity => " + testPlanRunEntity);
+        for (Key<TestRunEntity> testRunKey : testPlanRunEntity.getTestRuns()) {
             List<ApiCoverageEntity> apiCoverageEntityList =
                     ofy().load().type(ApiCoverageEntity.class).ancestor(testRunKey).list();
             for (ApiCoverageEntity apiCoverageEntity : apiCoverageEntityList) {
diff --git a/src/main/java/com/android/vts/api/DatastoreRestServlet.java b/src/main/java/com/android/vts/api/DatastoreRestServlet.java
index 42d7a60..c5c097f 100644
--- a/src/main/java/com/android/vts/api/DatastoreRestServlet.java
+++ b/src/main/java/com/android/vts/api/DatastoreRestServlet.java
@@ -16,58 +16,60 @@
 
 package com.android.vts.api;
 
+import com.android.vts.entity.ApiCoverageEntity;
+import com.android.vts.entity.BranchEntity;
+import com.android.vts.entity.BuildTargetEntity;
+import com.android.vts.entity.CodeCoverageEntity;
+import com.android.vts.entity.CoverageEntity;
+import com.android.vts.entity.DashboardEntity;
+import com.android.vts.entity.DeviceInfoEntity;
+import com.android.vts.entity.HalApiEntity;
+import com.android.vts.entity.ProfilingPointRunEntity;
+import com.android.vts.entity.TestCaseRunEntity;
+import com.android.vts.entity.TestEntity;
+import com.android.vts.entity.TestPlanEntity;
+import com.android.vts.entity.TestPlanRunEntity;
+import com.android.vts.entity.TestRunEntity;
+import com.android.vts.proto.VtsReportMessage;
 import com.android.vts.proto.VtsReportMessage.DashboardPostMessage;
 import com.android.vts.proto.VtsReportMessage.TestPlanReportMessage;
 import com.android.vts.proto.VtsReportMessage.TestReportMessage;
-import com.android.vts.servlet.BaseServlet;
-import com.android.vts.util.DatastoreHelper;
-import com.android.vts.util.EmailHelper;
 import com.google.api.client.googleapis.auth.oauth2.GoogleCredential;
 import com.google.api.client.http.javanet.NetHttpTransport;
 import com.google.api.client.json.jackson.JacksonFactory;
 import com.google.api.services.oauth2.Oauth2;
 import com.google.api.services.oauth2.model.Tokeninfo;
 
-import java.io.FileNotFoundException;
 import java.io.IOException;
-import java.io.InputStream;
-import java.util.Properties;
-import java.util.logging.Level;
-import java.util.logging.Logger;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
 import java.util.stream.Collectors;
 import javax.servlet.ServletConfig;
 import javax.servlet.ServletException;
-import javax.servlet.http.HttpServlet;
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
+
+import com.google.appengine.api.datastore.Key;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.codec.binary.Base64;
 
+import static com.googlecode.objectify.ObjectifyService.ofy;
+
+@Slf4j
 /** REST endpoint for posting data to the Dashboard. */
-public class DatastoreRestServlet extends HttpServlet {
+public class DatastoreRestServlet extends BaseApiServlet {
     private static String SERVICE_CLIENT_ID;
     private static final String SERVICE_NAME = "VTS Dashboard";
-    private static final Logger logger = Logger.getLogger(DatastoreRestServlet.class.getName());
-
-    /** System Configuration Property class */
-    protected Properties systemConfigProp = new Properties();
 
     @Override
     public void init(ServletConfig cfg) throws ServletException {
         super.init(cfg);
 
-        try {
-            InputStream defaultInputStream =
-                    DatastoreRestServlet.class
-                            .getClassLoader()
-                            .getResourceAsStream("config.properties");
-            systemConfigProp.load(defaultInputStream);
-
-            SERVICE_CLIENT_ID = systemConfigProp.getProperty("appengine.serviceClientID");
-        } catch (FileNotFoundException e) {
-            e.printStackTrace();
-        } catch (IOException e) {
-            e.printStackTrace();
-        }
+        SERVICE_CLIENT_ID = this.systemConfigProp.getProperty("appengine.serviceClientID");
     }
 
     @Override
@@ -81,14 +83,15 @@ public class DatastoreRestServlet extends HttpServlet {
             postMessage = DashboardPostMessage.parseFrom(value);
         } catch (IOException e) {
             response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
-            logger.log(Level.WARNING, "Invalid proto: " + e.getLocalizedMessage());
+            log.error("Invalid proto: " + e.getLocalizedMessage());
             return;
         }
 
         // Verify service account access token.
         if (postMessage.hasAccessToken()) {
             String accessToken = postMessage.getAccessToken();
-            logger.log(Level.INFO, "accessToken => " + accessToken);
+            log.debug("accessToken => " + accessToken);
+
             GoogleCredential credential = new GoogleCredential().setAccessToken(accessToken);
             Oauth2 oauth2 =
                     new Oauth2.Builder(new NetHttpTransport(), new JacksonFactory(), credential)
@@ -97,23 +100,461 @@ public class DatastoreRestServlet extends HttpServlet {
             Tokeninfo tokenInfo = oauth2.tokeninfo().setAccessToken(accessToken).execute();
             if (tokenInfo.getIssuedTo().equals(SERVICE_CLIENT_ID)) {
                 for (TestReportMessage testReportMessage : postMessage.getTestReportList()) {
-                    DatastoreHelper.insertTestReport(testReportMessage);
+                    this.insertTestReport(testReportMessage);
                 }
 
                 for (TestPlanReportMessage planReportMessage :
                         postMessage.getTestPlanReportList()) {
-                    DatastoreHelper.insertTestPlanReport(planReportMessage);
+                    this.insertTestPlanReport(planReportMessage);
                 }
 
                 response.setStatus(HttpServletResponse.SC_OK);
             } else {
-                logger.log(Level.WARNING, "service_client_id didn't match!");
-                logger.log(Level.INFO, "SERVICE_CLIENT_ID => " + tokenInfo.getIssuedTo());
+                log.warn("service_client_id didn't match!");
+                log.debug("SERVICE_CLIENT_ID => " + tokenInfo.getIssuedTo());
                 response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
             }
         } else {
-            logger.log(Level.WARNING, "postMessage do not contain any accessToken!");
+            log.error("postMessage do not contain any accessToken!");
             response.setStatus(HttpServletResponse.SC_UNAUTHORIZED);
         }
     }
+
+    /**
+     * Upload data from a test report message
+     *
+     * @param report The test report containing data to upload.
+     */
+    private void insertTestReport(TestReportMessage report) {
+
+        if (!report.hasStartTimestamp()
+                || !report.hasEndTimestamp()
+                || !report.hasTest()
+                || !report.hasHostInfo()
+                || !report.hasBuildInfo()) {
+            // missing information
+            log.error("Missing information in report !");
+            return;
+        }
+
+        List<TestEntity> testEntityList = new ArrayList<>();
+        List<TestRunEntity> testRunEntityList = new ArrayList<>();
+        List<BranchEntity> branchEntityList = new ArrayList<>();
+        List<BuildTargetEntity> buildTargetEntityList = new ArrayList<>();
+        List<CoverageEntity> coverageEntityList = new ArrayList<>();
+        List<CodeCoverageEntity> codeCoverageEntityList = new ArrayList<>();
+        List<DeviceInfoEntity> deviceInfoEntityList = new ArrayList<>();
+        List<ProfilingPointRunEntity> profilingPointRunEntityList = new ArrayList<>();
+        List<TestCaseRunEntity> testCaseRunEntityList = new ArrayList<>();
+        List<ApiCoverageEntity> apiCoverageEntityList = new ArrayList<>();
+
+        List<?> allEntityList =
+                Arrays.asList(
+                        testEntityList,
+                        branchEntityList,
+                        buildTargetEntityList,
+                        coverageEntityList,
+                        codeCoverageEntityList,
+                        deviceInfoEntityList,
+                        profilingPointRunEntityList,
+                        testCaseRunEntityList,
+                        apiCoverageEntityList,
+                        testRunEntityList);
+
+        long passCount = 0;
+        long failCount = 0;
+        long coveredLineCount = 0;
+        long totalLineCount = 0;
+
+        Set<Key> buildTargetKeys = new HashSet<>();
+        Set<Key> branchKeys = new HashSet<>();
+        List<Key> profilingPointKeyList = new ArrayList<>();
+        List<String> linkList = new ArrayList<>();
+
+        long startTimestamp = report.getStartTimestamp();
+        long endTimestamp = report.getEndTimestamp();
+        String testName = report.getTest().toStringUtf8();
+        String testBuildId = report.getBuildInfo().getId().toStringUtf8();
+        String hostName = report.getHostInfo().getHostname().toStringUtf8();
+
+        TestEntity testEntity = new TestEntity(testName);
+
+        com.googlecode.objectify.Key testRunKey =
+                testEntity.getTestRunKey(report.getStartTimestamp());
+
+        testEntityList.add(testEntity);
+
+        int testCaseRunEntityIndex = 0;
+        testCaseRunEntityList.add(new TestCaseRunEntity());
+        // Process test cases
+        for (VtsReportMessage.TestCaseReportMessage testCase : report.getTestCaseList()) {
+            String testCaseName = testCase.getName().toStringUtf8();
+            VtsReportMessage.TestCaseResult result = testCase.getTestResult();
+            // Track global pass/fail counts
+            if (result == VtsReportMessage.TestCaseResult.TEST_CASE_RESULT_PASS) {
+                ++passCount;
+            } else if (result != VtsReportMessage.TestCaseResult.TEST_CASE_RESULT_SKIP) {
+                ++failCount;
+            }
+            if (testCase.getSystraceCount() > 0
+                    && testCase.getSystraceList().get(0).getUrlCount() > 0) {
+                String systraceLink = testCase.getSystraceList().get(0).getUrl(0).toStringUtf8();
+                linkList.add(systraceLink);
+            }
+
+            // Process coverage data for test case
+            for (VtsReportMessage.CoverageReportMessage coverage : testCase.getCoverageList()) {
+                CoverageEntity coverageEntity =
+                        CoverageEntity.fromCoverageReport(testRunKey, testCaseName, coverage);
+                if (coverageEntity == null) {
+                    log.warn("Invalid coverage report in test run " + testRunKey);
+                } else {
+                    coveredLineCount += coverageEntity.getCoveredCount();
+                    totalLineCount += coverageEntity.getTotalCount();
+                    coverageEntityList.add(coverageEntity);
+                }
+            }
+
+            // Process profiling data for test case
+            for (VtsReportMessage.ProfilingReportMessage profiling : testCase.getProfilingList()) {
+                ProfilingPointRunEntity profilingPointRunEntity =
+                        ProfilingPointRunEntity.fromProfilingReport(testRunKey, profiling);
+                if (profilingPointRunEntity == null) {
+                    log.warn("Invalid profiling report in test run " + testRunKey);
+                } else {
+                    profilingPointRunEntityList.add(profilingPointRunEntity);
+                    profilingPointKeyList.add(profilingPointRunEntity.getKey());
+                    testEntity.setHasProfilingData(true);
+                }
+            }
+
+            TestCaseRunEntity testCaseRunEntity = testCaseRunEntityList.get(testCaseRunEntityIndex);
+            if (!testCaseRunEntity.addTestCase(testCaseName, result.getNumber())) {
+                testCaseRunEntity = new TestCaseRunEntity();
+                testCaseRunEntity.addTestCase(testCaseName, result.getNumber());
+                testCaseRunEntityList.add(testCaseRunEntity);
+                testCaseRunEntityIndex++;
+            }
+        }
+
+        // Process device information
+        long testRunType = 0;
+        for (VtsReportMessage.AndroidDeviceInfoMessage device : report.getDeviceInfoList()) {
+            DeviceInfoEntity deviceInfoEntity =
+                    DeviceInfoEntity.fromDeviceInfoMessage(testRunKey, device);
+            if (deviceInfoEntity == null) {
+                log.warn("Invalid device info in test run " + testRunKey);
+            } else {
+                // Run type on devices must be the same, else set to OTHER
+                TestRunEntity.TestRunType runType =
+                        TestRunEntity.TestRunType.fromBuildId(deviceInfoEntity.getBuildId());
+                if (runType == null) {
+                    testRunType = TestRunEntity.TestRunType.OTHER.getNumber();
+                } else {
+                    testRunType = runType.getNumber();
+                }
+                deviceInfoEntityList.add(deviceInfoEntity);
+                BuildTargetEntity target = new BuildTargetEntity(deviceInfoEntity.getBuildFlavor());
+                if (buildTargetKeys.add(target.getKey())) {
+                    buildTargetEntityList.add(target);
+                }
+                BranchEntity branch = new BranchEntity(deviceInfoEntity.getBranch());
+                if (branchKeys.add(branch.getKey())) {
+                    branchEntityList.add(branch);
+                }
+            }
+        }
+
+        // Overall run type should be determined by the device builds unless test build is OTHER
+        if (testRunType == TestRunEntity.TestRunType.OTHER.getNumber()) {
+            testRunType = TestRunEntity.TestRunType.fromBuildId(testBuildId).getNumber();
+        } else if (TestRunEntity.TestRunType.fromBuildId(testBuildId)
+                == TestRunEntity.TestRunType.OTHER) {
+            testRunType = TestRunEntity.TestRunType.OTHER.getNumber();
+        }
+
+        // Process global coverage data
+        for (VtsReportMessage.CoverageReportMessage coverage : report.getCoverageList()) {
+            CoverageEntity coverageEntity =
+                    CoverageEntity.fromCoverageReport(testRunKey, new String(), coverage);
+            if (coverageEntity == null) {
+                log.warn("Invalid coverage report in test run " + testRunKey);
+            } else {
+                coveredLineCount += coverageEntity.getCoveredCount();
+                totalLineCount += coverageEntity.getTotalCount();
+                coverageEntityList.add(coverageEntity);
+            }
+        }
+
+        // Process global API coverage data
+        for (VtsReportMessage.ApiCoverageReportMessage apiCoverage : report.getApiCoverageList()) {
+            VtsReportMessage.HalInterfaceMessage halInterfaceMessage =
+                    apiCoverage.getHalInterface();
+            List<String> halApiList =
+                    apiCoverage
+                            .getHalApiList()
+                            .stream()
+                            .map(h -> h.toStringUtf8())
+                            .collect(Collectors.toList());
+            List<String> coveredHalApiList =
+                    apiCoverage
+                            .getCoveredHalApiList()
+                            .stream()
+                            .map(h -> h.toStringUtf8())
+                            .collect(Collectors.toList());
+            ApiCoverageEntity apiCoverageEntity =
+                    new ApiCoverageEntity(
+                            testRunKey,
+                            halInterfaceMessage.getHalPackageName().toStringUtf8(),
+                            halInterfaceMessage.getHalVersionMajor(),
+                            halInterfaceMessage.getHalVersionMinor(),
+                            halInterfaceMessage.getHalInterfaceName().toStringUtf8(),
+                            halApiList,
+                            coveredHalApiList);
+            apiCoverageEntityList.add(apiCoverageEntity);
+        }
+
+        // Process global profiling data
+        for (VtsReportMessage.ProfilingReportMessage profiling : report.getProfilingList()) {
+            ProfilingPointRunEntity profilingPointRunEntity =
+                    ProfilingPointRunEntity.fromProfilingReport(testRunKey, profiling);
+            if (profilingPointRunEntity == null) {
+                log.warn("Invalid profiling report in test run " + testRunKey);
+            } else {
+                profilingPointRunEntityList.add(profilingPointRunEntity);
+                profilingPointKeyList.add(profilingPointRunEntity.getKey());
+                testEntity.setHasProfilingData(true);
+            }
+        }
+
+        // Process log data
+        for (VtsReportMessage.LogMessage log : report.getLogList()) {
+            if (log.hasUrl()) {
+                linkList.add(log.getUrl().toStringUtf8());
+            }
+        }
+        // Process url resource
+        for (VtsReportMessage.UrlResourceMessage resource : report.getLinkResourceList()) {
+            if (resource.hasUrl()) {
+                linkList.add(resource.getUrl().toStringUtf8());
+            }
+        }
+
+        boolean hasCodeCoverage = totalLineCount > 0 && coveredLineCount >= 0;
+        TestRunEntity testRunEntity =
+                new TestRunEntity(
+                        testEntity.getOldKey(),
+                        testRunType,
+                        startTimestamp,
+                        endTimestamp,
+                        testBuildId,
+                        hostName,
+                        passCount,
+                        failCount,
+                        hasCodeCoverage,
+                        new ArrayList<>(),
+                        linkList);
+        testRunEntityList.add(testRunEntity);
+
+        CodeCoverageEntity codeCoverageEntity =
+                new CodeCoverageEntity(
+                        testRunEntity.getId(),
+                        testRunEntity.getKey(),
+                        coveredLineCount,
+                        totalLineCount);
+        codeCoverageEntityList.add(codeCoverageEntity);
+
+        ofy().transact(
+                        () -> {
+                            List<Long> testCaseIds = new ArrayList<>();
+                            for (Object entity : allEntityList) {
+                                if (entity instanceof List) {
+                                    List listEntity = (List) entity;
+                                    if (listEntity.size() > 0
+                                            && listEntity.get(0) instanceof TestCaseRunEntity) {
+                                        List<TestCaseRunEntity> dashboardEntityList =
+                                                (List<TestCaseRunEntity>) entity;
+                                        Map<
+                                                        com.googlecode.objectify.Key<
+                                                                TestCaseRunEntity>,
+                                                        TestCaseRunEntity>
+                                                testCaseRunEntityMap =
+                                                        DashboardEntity.saveAll(
+                                                                dashboardEntityList,
+                                                                this
+                                                                        .MAX_ENTITY_SIZE_PER_TRANSACTION);
+
+                                        testCaseIds =
+                                                testCaseRunEntityMap
+                                                        .values()
+                                                        .stream()
+                                                        .map(
+                                                                testCaseRunEntity ->
+                                                                        testCaseRunEntity.getId())
+                                                        .collect(Collectors.toList());
+                                    } else if (listEntity.size() > 0
+                                            && listEntity.get(0) instanceof TestRunEntity) {
+                                        List<TestRunEntity> dashboardEntityList =
+                                                (List<TestRunEntity>) entity;
+                                        dashboardEntityList.get(0).setTestCaseIds(testCaseIds);
+                                        DashboardEntity.saveAll(
+                                                dashboardEntityList,
+                                                this.MAX_ENTITY_SIZE_PER_TRANSACTION);
+                                    } else {
+                                        List<DashboardEntity> dashboardEntityList =
+                                                (List<DashboardEntity>) entity;
+                                        DashboardEntity.saveAll(
+                                                dashboardEntityList,
+                                                this.MAX_ENTITY_SIZE_PER_TRANSACTION);
+                                    }
+                                }
+                            }
+                        });
+    }
+
+    /**
+     * Upload data from a test plan report message
+     *
+     * @param report The test plan report containing data to upload.
+     */
+    private void insertTestPlanReport(TestPlanReportMessage report) {
+        List<DeviceInfoEntity> deviceInfoEntityList = new ArrayList<>();
+        List<HalApiEntity> halApiEntityList = new ArrayList<>();
+
+        List allEntityList = Arrays.asList(deviceInfoEntityList, halApiEntityList);
+
+        List<String> testModules = report.getTestModuleNameList();
+        List<Long> testTimes = report.getTestModuleStartTimestampList();
+        if (testModules.size() != testTimes.size() || !report.hasTestPlanName()) {
+            log.error("TestPlanReportMessage is missing information.");
+            return;
+        }
+
+        String testPlanName = report.getTestPlanName();
+        TestPlanEntity testPlanEntity = new TestPlanEntity(testPlanName);
+        List<com.googlecode.objectify.Key<TestRunEntity>> testRunKeyList = new ArrayList<>();
+        for (int index = 0; index < testModules.size(); index++) {
+            String test = testModules.get(index);
+            long time = testTimes.get(index);
+            com.googlecode.objectify.Key testKey =
+                    com.googlecode.objectify.Key.create(TestEntity.class, test);
+            com.googlecode.objectify.Key testRunKey =
+                    com.googlecode.objectify.Key.create(testKey, TestRunEntity.class, time);
+            testRunKeyList.add(testRunKey);
+        }
+
+        Map<com.googlecode.objectify.Key<TestRunEntity>, TestRunEntity> testRunEntityMap =
+                ofy().load().keys(() -> testRunKeyList.iterator());
+
+        testRunKeyList.forEach(
+                (v) -> {
+                    log.debug("TestRunEntity key value => " + v);
+                });
+        log.debug("testRunEntityMap value => " + testRunEntityMap.values());
+        log.debug("testRunEntityMap keySet => " + testRunEntityMap.keySet());
+
+        long passCount = 0;
+        long failCount = 0;
+        long startTimestamp = -1;
+        long endTimestamp = -1;
+        String testBuildId = null;
+        long type = 0;
+        Set<DeviceInfoEntity> deviceInfoEntitySet = new HashSet<>();
+        for (TestRunEntity testRunEntity : testRunEntityMap.values()) {
+            passCount += testRunEntity.getPassCount();
+            failCount += testRunEntity.getFailCount();
+            if (startTimestamp < 0 || testRunEntity.getStartTimestamp() < startTimestamp) {
+                startTimestamp = testRunEntity.getStartTimestamp();
+            }
+            if (endTimestamp < 0 || testRunEntity.getEndTimestamp() > endTimestamp) {
+                endTimestamp = testRunEntity.getEndTimestamp();
+            }
+            type = testRunEntity.getType();
+            testBuildId = testRunEntity.getTestBuildId();
+
+            List<DeviceInfoEntity> deviceInfoEntityListWithTestRunKey =
+                    ofy().load()
+                            .type(DeviceInfoEntity.class)
+                            .ancestor(testRunEntity.getOfyKey())
+                            .list();
+
+            for (DeviceInfoEntity deviceInfoEntity : deviceInfoEntityListWithTestRunKey) {
+                deviceInfoEntitySet.add(deviceInfoEntity);
+            }
+        }
+
+        if (startTimestamp < 0 || testBuildId == null || type == 0) {
+            log.debug("startTimestamp => " + startTimestamp);
+            log.debug("testBuildId => " + testBuildId);
+            log.debug("type => " + type);
+            log.error("Couldn't infer test run information from runs.");
+            return;
+        }
+
+        TestPlanRunEntity testPlanRunEntity =
+                new TestPlanRunEntity(
+                        testPlanEntity.getKey(),
+                        testPlanName,
+                        type,
+                        startTimestamp,
+                        endTimestamp,
+                        testBuildId,
+                        passCount,
+                        failCount,
+                        0L,
+                        0L,
+                        testRunKeyList);
+
+        // Create the device infos.
+        for (DeviceInfoEntity device : deviceInfoEntitySet) {
+            deviceInfoEntityList.add(device.copyWithParent(testPlanRunEntity.getOfyKey()));
+        }
+
+        // Process global HAL API coverage data
+        for (VtsReportMessage.ApiCoverageReportMessage apiCoverage : report.getHalApiReportList()) {
+            VtsReportMessage.HalInterfaceMessage halInterfaceMessage =
+                    apiCoverage.getHalInterface();
+            List<String> halApiList =
+                    apiCoverage
+                            .getHalApiList()
+                            .stream()
+                            .map(h -> h.toStringUtf8())
+                            .collect(Collectors.toList());
+            List<String> coveredHalApiList =
+                    apiCoverage
+                            .getCoveredHalApiList()
+                            .stream()
+                            .map(h -> h.toStringUtf8())
+                            .collect(Collectors.toList());
+            HalApiEntity halApiEntity =
+                    new HalApiEntity(
+                            testPlanRunEntity.getParent(),
+                            halInterfaceMessage.getHalReleaseLevel().toStringUtf8(),
+                            halInterfaceMessage.getHalPackageName().toStringUtf8(),
+                            halInterfaceMessage.getHalVersionMajor(),
+                            halInterfaceMessage.getHalVersionMinor(),
+                            halInterfaceMessage.getHalInterfaceName().toStringUtf8(),
+                            halApiList,
+                            coveredHalApiList);
+            halApiEntityList.add(halApiEntity);
+        }
+
+        ofy().transact(
+                        () -> {
+                            testPlanEntity.save();
+                            testPlanRunEntity.save();
+                            for (Object entity : allEntityList) {
+                                List<DashboardEntity> dashboardEntityList =
+                                        (List<DashboardEntity>) entity;
+                                Map<com.googlecode.objectify.Key<DashboardEntity>, DashboardEntity>
+                                        mapInfo =
+                                                DashboardEntity.saveAll(
+                                                        dashboardEntityList,
+                                                        this.MAX_ENTITY_SIZE_PER_TRANSACTION);
+                            }
+                        });
+
+        // Add the task to calculate total number API list.
+        testPlanRunEntity.addCoverageApiTask();
+    }
 }
diff --git a/src/main/java/com/android/vts/api/TestDataForDevServlet.java b/src/main/java/com/android/vts/api/TestDataForDevServlet.java
index 7fc067f..2ed387b 100644
--- a/src/main/java/com/android/vts/api/TestDataForDevServlet.java
+++ b/src/main/java/com/android/vts/api/TestDataForDevServlet.java
@@ -490,14 +490,14 @@ public class TestDataForDevServlet extends HttpServlet {
                                                     BuildTargetEntity buildTargetEntity =
                                                             new BuildTargetEntity(
                                                                     buildTarget.targetName);
-                                                    datastore.put(buildTargetEntity.toEntity());
+                                                    buildTargetEntity.save();
                                                 });
 
                                         testRun.branchList.forEach(
                                                 branch -> {
                                                     BranchEntity branchEntity =
                                                             new BranchEntity(branch.branchName);
-                                                    datastore.put(branchEntity.toEntity());
+                                                    branchEntity.save();
                                                 });
 
                                         boolean hasCodeCoverage =
@@ -630,13 +630,14 @@ public class TestDataForDevServlet extends HttpServlet {
                                             testBuildId,
                                             passCount,
                                             failCount,
-                                        0L,
-                                        0L,
+                                            0L,
+                                            0L,
                                             testRunKeys);
 
                             // Create the device infos.
                             for (DeviceInfoEntity device : devices) {
-                                datastore.put(device.copyWithParent(testPlanRun.key).toEntity());
+                                datastore.put(
+                                        device.copyWithParent(testPlanRun.getOfyKey()).toEntity());
                             }
                             datastore.put(testPlanRun.toEntity());