diff options
| author | Young Gyu Park <younggyu@google.com> | 2018-07-06 11:03:54 +0900 |
|---|---|---|
| committer | Young Gyu Park <younggyu@google.com> | 2018-07-10 14:40:59 +0900 |
| commit | 1a483b6bad4e9effd2eeb3ab691f2ccfb0611d43 (patch) | |
| tree | db35679dd6a0cf4cc4ee79474ba019ff0546f56d /src/main/java | |
| parent | d747ca6e4b4f1e59c0e098dd33669a36f28ce00b (diff) | |
| download | dashboard-1a483b6bad4e9effd2eeb3ab691f2ccfb0611d43.tar.gz | |
Code coverage recalculation
Test: go/vts-web-staging
Bug: 111187342
Change-Id: I21c89cfb323ad5d432b38fc11cfcf0b1e78acab5
Diffstat (limited to 'src/main/java')
31 files changed, 2461 insertions, 1770 deletions
diff --git a/src/main/java/com/android/vts/api/BaseApiServlet.java b/src/main/java/com/android/vts/api/BaseApiServlet.java new file mode 100644 index 0000000..54a7574 --- /dev/null +++ b/src/main/java/com/android/vts/api/BaseApiServlet.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2018 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.android.vts.api; + +import com.google.apphosting.api.ApiProxy; +import com.google.gson.Gson; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; +import java.util.logging.Logger; +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * REST endpoint for posting test suite data to the Dashboard. + */ +public class BaseApiServlet extends HttpServlet { + + private static final Logger logger = + Logger.getLogger(BaseApiServlet.class.getName()); + + /** + * System Configuration Property class + */ + protected Properties systemConfigProp = new Properties(); + + /** + * Appengine server host name + */ + protected String hostName; + + @Override + public void init(ServletConfig cfg) throws ServletException { + super.init(cfg); + + ApiProxy.Environment env = ApiProxy.getCurrentEnvironment(); + hostName = env.getAttributes().get("com.google.appengine.runtime.default_version_hostname") + .toString(); + try { + InputStream defaultInputStream = + BaseApiServlet.class + .getClassLoader() + .getResourceAsStream("config.properties"); + systemConfigProp.load(defaultInputStream); + + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); + } + } + + protected void setAccessControlHeaders(HttpServletResponse resp) { + resp.setHeader("Access-Control-Allow-Origin", hostName); + resp.setHeader("Access-Control-Allow-Methods", "GET, PUT, POST, OPTIONS, DELETE"); + resp.addHeader("Access-Control-Allow-Headers", "Content-Type"); + resp.addHeader("Access-Control-Max-Age", "86400"); + } +} diff --git a/src/main/java/com/android/vts/api/CoverageRestServlet.java b/src/main/java/com/android/vts/api/CoverageRestServlet.java new file mode 100644 index 0000000..460949e --- /dev/null +++ b/src/main/java/com/android/vts/api/CoverageRestServlet.java @@ -0,0 +1,92 @@ +/* + * Copyright (c) 2018 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.android.vts.api; + +import com.android.vts.entity.CoverageEntity; +import com.android.vts.entity.TestCoverageStatusEntity; +import com.android.vts.entity.TestSuiteFileEntity; +import com.android.vts.entity.TestSuiteResultEntity; +import com.android.vts.proto.TestSuiteResultMessageProto.TestSuiteResultMessage; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.jackson.JacksonFactory; +import com.google.api.services.oauth2.Oauth2; +import com.google.api.services.oauth2.model.Tokeninfo; +import com.google.gson.Gson; +import com.googlecode.objectify.Key; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import org.apache.commons.codec.binary.Base64; + +/** + * REST endpoint for posting test suite data to the Dashboard. + */ +public class CoverageRestServlet extends BaseApiServlet { + + private static final Logger logger = + Logger.getLogger(CoverageRestServlet.class.getName()); + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws IOException { + + String cmd = request.getParameter("cmd"); + String coverageId = request.getParameter("coverageId"); + String testName = request.getParameter("testName"); + String testRunId = request.getParameter("testRunId"); + + Boolean isIgnored = false; + if (cmd.equals("disable")) { + isIgnored = true; + } + CoverageEntity coverageEntity = CoverageEntity.findById(testName, testRunId, coverageId); + coverageEntity.setIsIgnored(isIgnored); + coverageEntity.save(); + + TestCoverageStatusEntity testCoverageStatusEntity = TestCoverageStatusEntity.findById(testName); + Long newCoveredLineCount = + cmd.equals("disable") ? testCoverageStatusEntity.getUpdatedCoveredLineCount() + - coverageEntity.getCoveredCount() + : testCoverageStatusEntity.getUpdatedCoveredLineCount() + coverageEntity + .getCoveredCount(); + Long newTotalLineCount = + cmd.equals("disable") ? testCoverageStatusEntity.getUpdatedTotalLineCount() - coverageEntity + .getTotalCount() + : testCoverageStatusEntity.getUpdatedTotalLineCount() + coverageEntity.getTotalCount(); + testCoverageStatusEntity.setUpdatedCoveredLineCount(newCoveredLineCount); + testCoverageStatusEntity.setUpdatedTotalLineCount(newTotalLineCount); + testCoverageStatusEntity.save(); + + String json = new Gson().toJson("Success!"); + response.setStatus(HttpServletResponse.SC_OK); + response.setContentType("application/json"); + response.setCharacterEncoding("UTF-8"); + response.getWriter().write(json); + } +} diff --git a/src/main/java/com/android/vts/api/TestDataForDevServlet.java b/src/main/java/com/android/vts/api/TestDataForDevServlet.java index 75432c7..70aa74a 100644 --- a/src/main/java/com/android/vts/api/TestDataForDevServlet.java +++ b/src/main/java/com/android/vts/api/TestDataForDevServlet.java @@ -402,7 +402,7 @@ public class TestDataForDevServlet extends HttpServlet { Key testRunKey = KeyFactory.createKey( - testEntity.key, + testEntity.getOldKey(), TestRunEntity.KIND, testRun.startTimestamp); @@ -504,7 +504,7 @@ public class TestDataForDevServlet extends HttpServlet { TestRunEntity testRunEntity = new TestRunEntity( - testEntity.key, + testEntity.getOldKey(), TestRunType.fromNumber(testRun.type), testRun.startTimestamp, testRun.endTimestamp, @@ -524,7 +524,7 @@ public class TestDataForDevServlet extends HttpServlet { try { // Check if test already exists in the datastore try { - Entity oldTest = datastore.get(testEntity.key); + Entity oldTest = datastore.get(testEntity.getOldKey()); TestEntity oldTestEntity = TestEntity.fromEntity(oldTest); if (oldTestEntity == null @@ -548,7 +548,7 @@ public class TestDataForDevServlet extends HttpServlet { logger.log( Level.WARNING, "Transaction rollback forced for run: " - + testRunEntity.key); + + testRunEntity.getKey()); txn.rollback(); } } @@ -584,20 +584,20 @@ public class TestDataForDevServlet extends HttpServlet { if (testRun == null) { continue; // not a valid test run } - passCount += testRun.passCount; - failCount += testRun.failCount; + passCount += testRun.getPassCount(); + failCount += testRun.getFailCount(); if (startTimestamp < 0 || testRunKey.getId() < startTimestamp) { startTimestamp = testRunKey.getId(); } - if (endTimestamp < 0 || testRun.endTimestamp > endTimestamp) { - endTimestamp = testRun.endTimestamp; + if (endTimestamp < 0 || testRun.getEndTimestamp() > endTimestamp) { + endTimestamp = testRun.getEndTimestamp(); } if (type == null) { - type = testRun.type; - } else if (type != testRun.type) { + type = testRun.getType(); + } else if (type != testRun.getType()) { type = TestRunType.OTHER; } - testBuildId = testRun.testBuildId; + testBuildId = testRun.getTestBuildId(); Query deviceInfoQuery = new Query(DeviceInfoEntity.KIND).setAncestor(testRunKey); for (Entity deviceInfoEntity : diff --git a/src/main/java/com/android/vts/api/TestRunRestServlet.java b/src/main/java/com/android/vts/api/TestRunRestServlet.java index faadb61..67fd684 100644 --- a/src/main/java/com/android/vts/api/TestRunRestServlet.java +++ b/src/main/java/com/android/vts/api/TestRunRestServlet.java @@ -77,7 +77,7 @@ public class TestRunRestServlet extends HttpServlet { } TestRunDetails details = new TestRunDetails(); List<Key> gets = new ArrayList<>(); - for (long testCaseId : testRunEntity.testCaseIds) { + for (long testCaseId : testRunEntity.getTestCaseIds()) { gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); } Map<Key, Entity> entityMap = datastore.get(gets); @@ -115,7 +115,7 @@ public class TestRunRestServlet extends HttpServlet { TestRunDetails details = new TestRunDetails(); List<Key> gets = new ArrayList<>(); - for (long testCaseId : testRun.testCaseIds) { + for (long testCaseId : testRun.getTestCaseIds()) { gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); } Map<Key, Entity> entityMap = datastore.get(gets); diff --git a/src/main/java/com/android/vts/api/TestSuiteResultRestServlet.java b/src/main/java/com/android/vts/api/TestSuiteResultRestServlet.java index 9887ccd..8f52c85 100644 --- a/src/main/java/com/android/vts/api/TestSuiteResultRestServlet.java +++ b/src/main/java/com/android/vts/api/TestSuiteResultRestServlet.java @@ -47,32 +47,17 @@ import javax.servlet.ServletConfig; import javax.servlet.ServletException; /** REST endpoint for posting test suite data to the Dashboard. */ -public class TestSuiteResultRestServlet extends HttpServlet { +public class TestSuiteResultRestServlet extends BaseApiServlet { private static String SERVICE_CLIENT_ID; private static final String SERVICE_NAME = "VTS Dashboard"; private static final Logger logger = Logger.getLogger(TestSuiteResultRestServlet.class.getName()); - /** System Configuration Property class */ - protected Properties systemConfigProp = new Properties(); - @Override public void init(ServletConfig cfg) throws ServletException { super.init(cfg); - try { - InputStream defaultInputStream = - TestSuiteResultRestServlet.class - .getClassLoader() - .getResourceAsStream("config.properties"); - systemConfigProp.load(defaultInputStream); - - SERVICE_CLIENT_ID = systemConfigProp.getProperty("appengine.serviceClientID"); - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } + SERVICE_CLIENT_ID = systemConfigProp.getProperty("appengine.serviceClientID"); } @Override diff --git a/src/main/java/com/android/vts/config/ObjectifyListener.java b/src/main/java/com/android/vts/config/ObjectifyListener.java index 835cba9..15b47f8 100644 --- a/src/main/java/com/android/vts/config/ObjectifyListener.java +++ b/src/main/java/com/android/vts/config/ObjectifyListener.java @@ -16,12 +16,23 @@ package com.android.vts.config; +import com.android.vts.entity.CoverageEntity; +import com.android.vts.entity.RoleEntity; +import com.android.vts.entity.TestCoverageStatusEntity; +import com.android.vts.entity.TestEntity; +import com.android.vts.entity.TestPlanEntity; +import com.android.vts.entity.TestPlanRunEntity; +import com.android.vts.entity.TestRunEntity; import com.android.vts.entity.TestSuiteFileEntity; import com.android.vts.entity.TestSuiteResultEntity; import com.android.vts.entity.UserEntity; import com.googlecode.objectify.Key; import com.googlecode.objectify.ObjectifyService; +import java.util.Arrays; +import java.util.Optional; +import java.util.function.Supplier; +import java.util.stream.Stream; import javax.servlet.ServletContext; import javax.servlet.ServletContextEvent; import javax.servlet.ServletContextListener; @@ -36,7 +47,9 @@ import java.util.logging.Logger; import static com.googlecode.objectify.ObjectifyService.ofy; -/** The @WebListener annotation for registering a class as a listener of a web application. */ +/** + * The @WebListener annotation for registering a class as a listener of a web application. + */ @WebListener /** * Initializing Objectify Service at the container start up before any web components like servlet @@ -44,56 +57,84 @@ import static com.googlecode.objectify.ObjectifyService.ofy; */ public class ObjectifyListener implements ServletContextListener { - private static final Logger logger = Logger.getLogger(ObjectifyListener.class.getName()); + private static final Logger logger = Logger.getLogger(ObjectifyListener.class.getName()); - /** - * Receives notification that the web application initialization process is starting. This - * function will register Entity classes for objectify. - */ - @Override - public void contextInitialized(ServletContextEvent servletContextEvent) { - ObjectifyService.init(); - ObjectifyService.register(TestSuiteFileEntity.class); - ObjectifyService.register(TestSuiteResultEntity.class); - ObjectifyService.register(UserEntity.class); - ObjectifyService.begin(); - logger.log(Level.INFO, "Value Initialized from context."); + /** + * Receives notification that the web application initialization process is starting. This + * function will register Entity classes for objectify. + */ + @Override + public void contextInitialized(ServletContextEvent servletContextEvent) { + ObjectifyService.init(); + ObjectifyService.register(CoverageEntity.class); + ObjectifyService.register(TestCoverageStatusEntity.class); + ObjectifyService.register(TestEntity.class); + ObjectifyService.register(TestPlanEntity.class); + ObjectifyService.register(TestPlanRunEntity.class); + ObjectifyService.register(TestRunEntity.class); + ObjectifyService.register(TestSuiteFileEntity.class); + ObjectifyService.register(TestSuiteResultEntity.class); + ObjectifyService.register(RoleEntity.class); + ObjectifyService.register(UserEntity.class); + ObjectifyService.begin(); + logger.log(Level.INFO, "Value Initialized from context."); - Properties systemConfigProp = new Properties(); + Properties systemConfigProp = new Properties(); - try { - InputStream defaultInputStream = - ObjectifyListener.class - .getClassLoader() - .getResourceAsStream("config.properties"); + try { + InputStream defaultInputStream = + ObjectifyListener.class + .getClassLoader() + .getResourceAsStream("config.properties"); - systemConfigProp.load(defaultInputStream); + systemConfigProp.load(defaultInputStream); - String adminEmail = systemConfigProp.getProperty("user.adminEmail"); - if (adminEmail.isEmpty()) { - logger.log(Level.WARNING, "Admin email is not properly set. Check config file"); - } else { - String adminName = systemConfigProp.getProperty("user.adminName"); - String adminCompany = systemConfigProp.getProperty("user.adminCompany"); + String roleList = systemConfigProp.getProperty("user.roleList"); + Supplier<Stream<String>> streamSupplier = () -> Arrays.stream(roleList.split(",")); + this.createRoles(streamSupplier.get()); - if (UserEntity.getAdminUserList(adminEmail).size() == 0) { - UserEntity userEntity = new UserEntity(adminEmail, adminName, adminCompany); - userEntity.setIsAdmin(true); - userEntity.save(); - logger.log(Level.INFO, "The user is saved successfully."); - } - } - } catch (FileNotFoundException e) { - e.printStackTrace(); - } catch (IOException e) { - e.printStackTrace(); - } + String adminEmail = systemConfigProp.getProperty("user.adminEmail"); + if (adminEmail.isEmpty()) { + logger.log(Level.WARNING, "Admin email is not properly set. Check config file"); + } else { + String adminName = systemConfigProp.getProperty("user.adminName"); + String adminCompany = systemConfigProp.getProperty("user.adminCompany"); + Optional<String> roleName = streamSupplier.get().filter(r -> r.equals("admin")).findFirst(); + this.createAdminUser(adminEmail, adminName, adminCompany, roleName.orElse("admin")); + } + } catch (FileNotFoundException e) { + e.printStackTrace(); + } catch (IOException e) { + e.printStackTrace(); } + } + + /** + * Receives notification that the ServletContext is about to be shut down. + */ + @Override + public void contextDestroyed(ServletContextEvent servletContextEvent) { + ServletContext servletContext = servletContextEvent.getServletContext(); + logger.log(Level.INFO, "Value deleted from context."); + } + + private void createRoles(Stream<String> roleStream) { + roleStream.map(role -> role.trim()).forEach(roleName -> { + RoleEntity roleEntity = new RoleEntity(roleName); + roleEntity.save(); + }); + } - /** Receives notification that the ServletContext is about to be shut down. */ - @Override - public void contextDestroyed(ServletContextEvent servletContextEvent) { - ServletContext servletContext = servletContextEvent.getServletContext(); - logger.log(Level.INFO, "Value deleted from context."); + private void createAdminUser(String email, String name, String company, String role) { + Optional<UserEntity> adminUserEntityOptional = Optional + .ofNullable(UserEntity.getAdminUser(email)); + if (adminUserEntityOptional.isPresent()) { + UserEntity userEntity = new UserEntity(email, name, company, role); + userEntity.setIsAdmin(true); + userEntity.save(); + logger.log(Level.INFO, "The user is saved successfully."); + } else { + logger.log(Level.INFO, "The user is already registered."); } + } } diff --git a/src/main/java/com/android/vts/entity/CoverageEntity.java b/src/main/java/com/android/vts/entity/CoverageEntity.java index a0f8cca..2554db9 100644 --- a/src/main/java/com/android/vts/entity/CoverageEntity.java +++ b/src/main/java/com/android/vts/entity/CoverageEntity.java @@ -16,141 +16,263 @@ package com.android.vts.entity; +import static com.googlecode.objectify.ObjectifyService.ofy; + import com.android.vts.proto.VtsReportMessage.CoverageReportMessage; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.Key; +import com.google.cloud.datastore.PathElement; +import com.googlecode.objectify.LoadResult; +import com.googlecode.objectify.annotation.Cache; +import com.googlecode.objectify.annotation.Id; +import com.googlecode.objectify.annotation.Ignore; +import com.googlecode.objectify.annotation.Index; +import com.googlecode.objectify.annotation.Parent; +import java.io.Serializable; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; +import java.util.Objects; +import java.util.Properties; import java.util.logging.Level; import java.util.logging.Logger; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +@com.googlecode.objectify.annotation.Entity(name = "Coverage") +@Cache +@Data +@NoArgsConstructor /** Object describing coverage data gathered for a file. */ -public class CoverageEntity implements DashboardEntity { - protected static final Logger logger = Logger.getLogger(CoverageEntity.class.getName()); - - public static final String KIND = "Coverage"; - - // Property keys - public static final String GROUP = "group"; - public static final String COVERED_LINE_COUNT = "coveredCount"; - public static final String TOTAL_LINE_COUNT = "totalCount"; - public static final String FILE_PATH = "filePath"; - public static final String PROJECT_NAME = "projectName"; - public static final String PROJECT_VERSION = "projectVersion"; - public static final String LINE_COVERAGE = "lineCoverage"; - - private final Key parentKey; - - public final String group; - public final long coveredLineCount; - public final long totalLineCount; - public final String filePath; - public final String projectName; - public final String projectVersion; - public final List<Long> lineCoverage; - - /** - * Create a CoverageEntity object for a file. - * - * @param parentKey The key to the parent TestRunEntity object in the database. - * @param group The group within the test run describing the coverage. - * @param coveredLineCount The total number of covered lines in the file. - * @param totalLineCount The total number of uncovered executable lines in the file. - * @param filePath The path to the file. - * @param projectName The name of the git project. - * @param projectVersion The commit hash of the project at the time the test was executed. - * @param lineCoverage List of coverage counts per executable line in the file. - */ - public CoverageEntity(Key parentKey, String group, long coveredLineCount, long totalLineCount, - String filePath, String projectName, String projectVersion, List<Long> lineCoverage) { - this.parentKey = parentKey; - this.group = group; - this.coveredLineCount = coveredLineCount; - this.totalLineCount = totalLineCount; - this.filePath = filePath; - this.projectName = projectName; - this.projectVersion = projectVersion; - this.lineCoverage = lineCoverage; +public class CoverageEntity implements Serializable { + + protected static final Logger logger = Logger.getLogger(CoverageEntity.class.getName()); + + public static final String KIND = "Coverage"; + + public static String GERRIT_URI; + + // Property keys + public static final String GROUP = "group"; + public static final String COVERED_LINE_COUNT = "coveredCount"; + public static final String TOTAL_LINE_COUNT = "totalCount"; + public static final String FILE_PATH = "filePath"; + public static final String PROJECT_NAME = "projectName"; + public static final String PROJECT_VERSION = "projectVersion"; + public static final String LINE_COVERAGE = "lineCoverage"; + + @Ignore + @Getter + @Setter + private Key parentKey; + + @Id + @Getter + @Setter + private Long ID; + + @Parent + @Getter + @Setter + private com.googlecode.objectify.Key<?> testParent; + + @Index + @Getter + @Setter + private String group; + + @Getter + @Setter + private long coveredCount; + + @Getter + @Setter + private long totalCount; + + @Index + @Getter + @Setter + private String filePath; + + @Getter + @Setter + private String projectName; + + @Getter + @Setter + private String projectVersion; + + @Getter + @Setter + private List<Long> lineCoverage; + + /** + * CoverageEntity isIgnored field + */ + @Index + @Getter + @Setter + Boolean isIgnored; + + /** + * Create a CoverageEntity object for a file. + * + * @param parentKey The key to the parent TestRunEntity object in the database. + * @param group The group within the test run describing the coverage. + * @param coveredLineCount The total number of covered lines in the file. + * @param totalLineCount The total number of uncovered executable lines in the file. + * @param filePath The path to the file. + * @param projectName The name of the git project. + * @param projectVersion The commit hash of the project at the time the test was executed. + * @param lineCoverage List of coverage counts per executable line in the file. + */ + public CoverageEntity(Key parentKey, String group, long coveredLineCount, long totalLineCount, + String filePath, String projectName, String projectVersion, List<Long> lineCoverage) { + this.parentKey = parentKey; + this.group = group; + this.coveredCount = coveredLineCount; + this.totalCount = totalLineCount; + this.filePath = filePath; + this.projectName = projectName; + this.projectVersion = projectVersion; + this.lineCoverage = lineCoverage; + } + + /** + * find coverage entity by ID + */ + public static CoverageEntity findById(String testName, String testRunId, String id) { + com.googlecode.objectify.Key testKey = com.googlecode.objectify.Key + .create(TestEntity.class, testName); + com.googlecode.objectify.Key testRunKey = com.googlecode.objectify.Key + .create(testKey, TestRunEntity.class, Long.parseLong(testRunId)); + return ofy().load().type(CoverageEntity.class).parent(testRunKey).id(Long.parseLong(id)).now(); + } + + public static void setPropertyValues(Properties newSystemConfigProp) { + GERRIT_URI = newSystemConfigProp.getProperty("gerrit.uri"); + } + + /** + * Saving function for the instance of this class + */ + public void save() { + ofy().save().entity(this).now(); + } + + /** + * Get percentage from calculating coveredCount and totalCount values + */ + public Double getPercentage() { + return Math.round(coveredCount * 10000d / totalCount) / 100d; + } + + /** + * Get Gerrit Url function from the attributes of this class + */ + public String getGerritUrl() throws UnsupportedEncodingException { + String gerritPath = GERRIT_URI + "/projects/" + + projectName + "/commits/" + + URLEncoder.encode(projectVersion, "UTF-8") + "/files/" + + filePath; + return gerritPath; + } + + /* Comparator for sorting the list by isIgnored field */ + public static Comparator<CoverageEntity> isIgnoredComparator = new Comparator<CoverageEntity>() { + + public int compare(CoverageEntity coverageEntity1, CoverageEntity coverageEntity2) { + Boolean isIgnored1 = + Objects.isNull(coverageEntity1.getIsIgnored()) ? false : coverageEntity1.getIsIgnored(); + Boolean isIgnored2 = + Objects.isNull(coverageEntity2.getIsIgnored()) ? false : coverageEntity2.getIsIgnored(); + + // ascending order + return isIgnored1.compareTo(isIgnored2); } + }; - @Override - public Entity toEntity() { - Entity coverageEntity = new Entity(KIND, parentKey); - coverageEntity.setProperty(GROUP, group); - coverageEntity.setUnindexedProperty(COVERED_LINE_COUNT, coveredLineCount); - coverageEntity.setUnindexedProperty(TOTAL_LINE_COUNT, totalLineCount); - coverageEntity.setProperty(FILE_PATH, filePath); - coverageEntity.setUnindexedProperty(PROJECT_NAME, projectName); - coverageEntity.setUnindexedProperty(PROJECT_VERSION, projectVersion); - if (lineCoverage != null && lineCoverage.size() > 0) { - coverageEntity.setUnindexedProperty(LINE_COVERAGE, lineCoverage); - } - return coverageEntity; + public Entity toEntity() { + Entity coverageEntity = new Entity(KIND, parentKey); + coverageEntity.setProperty(GROUP, group); + coverageEntity.setUnindexedProperty(COVERED_LINE_COUNT, coveredCount); + coverageEntity.setUnindexedProperty(TOTAL_LINE_COUNT, totalCount); + coverageEntity.setProperty(FILE_PATH, filePath); + coverageEntity.setUnindexedProperty(PROJECT_NAME, projectName); + coverageEntity.setUnindexedProperty(PROJECT_VERSION, projectVersion); + if (lineCoverage != null && lineCoverage.size() > 0) { + coverageEntity.setUnindexedProperty(LINE_COVERAGE, lineCoverage); } + return coverageEntity; + } - /** - * Convert an Entity object to a CoverageEntity. - * - * @param e The entity to process. - * @return CoverageEntity object with the properties from e, or null if incompatible. - */ - @SuppressWarnings("unchecked") - public static CoverageEntity fromEntity(Entity e) { - if (!e.getKind().equals(KIND) || !e.hasProperty(GROUP) || !e.hasProperty(COVERED_LINE_COUNT) - || !e.hasProperty(TOTAL_LINE_COUNT) || !e.hasProperty(FILE_PATH) - || !e.hasProperty(PROJECT_NAME) || !e.hasProperty(PROJECT_VERSION)) { - logger.log(Level.WARNING, "Missing coverage attributes in entity: " + e.toString()); - return null; - } - try { - String group = (String) e.getProperty(GROUP); - long coveredLineCount = (long) e.getProperty(COVERED_LINE_COUNT); - long totalLineCount = (long) e.getProperty(TOTAL_LINE_COUNT); - String filePath = (String) e.getProperty(FILE_PATH); - String projectName = (String) e.getProperty(PROJECT_NAME); - String projectVersion = (String) e.getProperty(PROJECT_VERSION); - List<Long> lineCoverage; - if (e.hasProperty(LINE_COVERAGE)) { - lineCoverage = (List<Long>) e.getProperty(LINE_COVERAGE); - } else { - lineCoverage = new ArrayList<>(); - } - return new CoverageEntity(e.getKey().getParent(), group, coveredLineCount, - totalLineCount, filePath, projectName, projectVersion, lineCoverage); - } catch (ClassCastException exception) { - // Invalid contents or null values - logger.log(Level.WARNING, "Error parsing coverage entity.", exception); - } - return null; + /** + * Convert an Entity object to a CoverageEntity. + * + * @param e The entity to process. + * @return CoverageEntity object with the properties from e, or null if incompatible. + */ + @SuppressWarnings("unchecked") + public static CoverageEntity fromEntity(Entity e) { + if (!e.getKind().equals(KIND) || !e.hasProperty(GROUP) || !e.hasProperty(COVERED_LINE_COUNT) + || !e.hasProperty(TOTAL_LINE_COUNT) || !e.hasProperty(FILE_PATH) + || !e.hasProperty(PROJECT_NAME) || !e.hasProperty(PROJECT_VERSION)) { + logger.log(Level.WARNING, "Missing coverage attributes in entity: " + e.toString()); + return null; + } + try { + String group = (String) e.getProperty(GROUP); + long coveredLineCount = (long) e.getProperty(COVERED_LINE_COUNT); + long totalLineCount = (long) e.getProperty(TOTAL_LINE_COUNT); + String filePath = (String) e.getProperty(FILE_PATH); + String projectName = (String) e.getProperty(PROJECT_NAME); + String projectVersion = (String) e.getProperty(PROJECT_VERSION); + List<Long> lineCoverage; + if (e.hasProperty(LINE_COVERAGE)) { + lineCoverage = (List<Long>) e.getProperty(LINE_COVERAGE); + } else { + lineCoverage = new ArrayList<>(); + } + return new CoverageEntity(e.getKey().getParent(), group, coveredLineCount, + totalLineCount, filePath, projectName, projectVersion, lineCoverage); + } catch (ClassCastException exception) { + // Invalid contents or null values + logger.log(Level.WARNING, "Error parsing coverage entity.", exception); } + return null; + } - /** - * Convert a coverage report to a CoverageEntity. - * - * @param parentKey The ancestor key for the coverage entity. - * @param group The group to display the coverage report with. - * @param coverage The coverage report containing coverage data. - * @return The CoverageEntity for the coverage report message, or null if not compatible. - */ - public static CoverageEntity fromCoverageReport( - Key parentKey, String group, CoverageReportMessage coverage) { - if (!coverage.hasFilePath() || !coverage.hasProjectName() || !coverage.hasRevision() - || !coverage.hasTotalLineCount() || !coverage.hasCoveredLineCount()) { - return null; // invalid coverage report; - } - long coveredLineCount = coverage.getCoveredLineCount(); - long totalLineCount = coverage.getTotalLineCount(); - String filePath = coverage.getFilePath().toStringUtf8(); - String projectName = coverage.getProjectName().toStringUtf8(); - String projectVersion = coverage.getRevision().toStringUtf8(); - List<Long> lineCoverage = null; - if (coverage.getLineCoverageVectorCount() > 0) { - lineCoverage = new ArrayList<>(); - for (int count : coverage.getLineCoverageVectorList()) { - lineCoverage.add((long) count); - } - } - return new CoverageEntity(parentKey, group, coveredLineCount, totalLineCount, filePath, - projectName, projectVersion, lineCoverage); + /** + * Convert a coverage report to a CoverageEntity. + * + * @param parentKey The ancestor key for the coverage entity. + * @param group The group to display the coverage report with. + * @param coverage The coverage report containing coverage data. + * @return The CoverageEntity for the coverage report message, or null if not compatible. + */ + public static CoverageEntity fromCoverageReport( + Key parentKey, String group, CoverageReportMessage coverage) { + if (!coverage.hasFilePath() || !coverage.hasProjectName() || !coverage.hasRevision() + || !coverage.hasTotalLineCount() || !coverage.hasCoveredLineCount()) { + return null; // invalid coverage report; + } + long coveredLineCount = coverage.getCoveredLineCount(); + long totalLineCount = coverage.getTotalLineCount(); + String filePath = coverage.getFilePath().toStringUtf8(); + String projectName = coverage.getProjectName().toStringUtf8(); + String projectVersion = coverage.getRevision().toStringUtf8(); + List<Long> lineCoverage = null; + if (coverage.getLineCoverageVectorCount() > 0) { + lineCoverage = new ArrayList<>(); + for (int count : coverage.getLineCoverageVectorList()) { + lineCoverage.add((long) count); + } } + return new CoverageEntity(parentKey, group, coveredLineCount, totalLineCount, filePath, + projectName, projectVersion, lineCoverage); + } } diff --git a/src/main/java/com/android/vts/entity/RoleEntity.java b/src/main/java/com/android/vts/entity/RoleEntity.java new file mode 100644 index 0000000..d001cfa --- /dev/null +++ b/src/main/java/com/android/vts/entity/RoleEntity.java @@ -0,0 +1,54 @@ +package com.android.vts.entity; + +import static com.googlecode.objectify.ObjectifyService.ofy; + +import com.googlecode.objectify.Key; +import com.googlecode.objectify.annotation.Cache; +import com.googlecode.objectify.annotation.Entity; +import com.googlecode.objectify.annotation.Id; +import com.googlecode.objectify.annotation.Index; +import com.googlecode.objectify.annotation.Load; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; + + +@Cache +@Entity +@EqualsAndHashCode(of = "role") +@NoArgsConstructor +public class RoleEntity implements Serializable { + + private static final long serialVersionUID = 1L; + + @Id + private String role; + + /** When this record was created or updated */ + @Getter + Date updated; + + /** Construction function for UserEntity Class */ + public RoleEntity(String roleName) { + this.role = roleName; + } + + /** Get role by email */ + public static RoleEntity getRole(String role) { + return ofy().load() + .type(RoleEntity.class) + .id(role) + .now(); + } + + /** Saving function for the instance of this class */ + public void save() { + this.updated = new Date(); + ofy().save().entity(this).now(); + } +}
\ No newline at end of file diff --git a/src/main/java/com/android/vts/entity/TestCoverageStatusEntity.java b/src/main/java/com/android/vts/entity/TestCoverageStatusEntity.java index afcf1c9..a9c8deb 100644 --- a/src/main/java/com/android/vts/entity/TestCoverageStatusEntity.java +++ b/src/main/java/com/android/vts/entity/TestCoverageStatusEntity.java @@ -16,82 +16,187 @@ package com.android.vts.entity; +import static com.googlecode.objectify.ObjectifyService.ofy; + import com.google.appengine.api.datastore.Entity; +import com.googlecode.objectify.annotation.Cache; +import com.googlecode.objectify.annotation.Id; +import com.googlecode.objectify.annotation.Index; +import java.io.Serializable; +import java.util.Date; +import java.util.List; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +@Cache +@com.googlecode.objectify.annotation.Entity(name = "TestCoverageStatus") +@EqualsAndHashCode(of = "testName") +@NoArgsConstructor /** Entity describing test coverage status. */ -public class TestCoverageStatusEntity implements DashboardEntity { - protected static final Logger logger = - Logger.getLogger(TestCoverageStatusEntity.class.getName()); - - public static final String KIND = "TestCoverageStatus"; - - // Property keys - public static final String TOTAL_LINE_COUNT = "totalLineCount"; - public static final String COVERED_LINE_COUNT = "coveredLineCount"; - public static final String UPDATED_TIMESTAMP = "updatedTimestamp"; - - public final String testName; - public final long coveredLineCount; - public final long totalLineCount; - public final long timestamp; - - /** - * Create a TestCoverageStatusEntity object with status metadata. - * - * @param testName The name of the test. - * @param timestamp The timestamp indicating the most recent test run event in the test state. - * @param coveredLineCount The number of lines covered. - * @param totalLineCount The total number of lines. - */ - public TestCoverageStatusEntity( - String testName, long timestamp, long coveredLineCount, long totalLineCount) { - this.testName = testName; - this.timestamp = timestamp; - this.coveredLineCount = coveredLineCount; - this.totalLineCount = totalLineCount; - } +public class TestCoverageStatusEntity implements Serializable { - @Override - public Entity toEntity() { - Entity testEntity = new Entity(KIND, this.testName); - testEntity.setProperty(UPDATED_TIMESTAMP, this.timestamp); - testEntity.setProperty(COVERED_LINE_COUNT, this.coveredLineCount); - testEntity.setProperty(TOTAL_LINE_COUNT, this.totalLineCount); - return testEntity; - } + protected static final Logger logger = + Logger.getLogger(TestCoverageStatusEntity.class.getName()); + + public static final String KIND = "TestCoverageStatus"; + + // Property keys + public static final String TOTAL_LINE_COUNT = "totalLineCount"; + public static final String COVERED_LINE_COUNT = "coveredLineCount"; + public static final String UPDATED_TIMESTAMP = "updatedTimestamp"; + + /** + * TestCoverageStatusEntity name field + */ + @Id + @Getter + @Setter + String testName; + + /** + * TestCoverageStatusEntity coveredLineCount field + */ + @Index + @Getter + @Setter + long coveredLineCount; + + /** + * TestCoverageStatusEntity totalLineCount field + */ + @Index + @Getter + @Setter + long totalLineCount; + + /** + * TestCoverageStatusEntity updatedTimestamp field + */ + @Index + @Getter + @Setter + long updatedTimestamp; + + /** + * TestCoverageStatusEntity updatedCoveredLineCount field + */ + @Index + @Getter + @Setter + long updatedCoveredLineCount; - /** - * Convert an Entity object to a TestCoverageStatusEntity. - * - * @param e The entity to process. - * @return TestCoverageStatusEntity object with the properties from e processed, or null if - * incompatible. - */ - @SuppressWarnings("unchecked") - public static TestCoverageStatusEntity fromEntity(Entity e) { - if (!e.getKind().equals(KIND) - || e.getKey().getName() == null - || !e.hasProperty(UPDATED_TIMESTAMP) - || !e.hasProperty(COVERED_LINE_COUNT) - || !e.hasProperty(TOTAL_LINE_COUNT)) { - logger.log(Level.WARNING, "Missing test attributes in entity: " + e.toString()); - return null; - } - String testName = e.getKey().getName(); - long timestamp = 0; - long coveredLineCount = -1; - long totalLineCount = -1; - try { - timestamp = (long) e.getProperty(UPDATED_TIMESTAMP); - coveredLineCount = (Long) e.getProperty(COVERED_LINE_COUNT); - totalLineCount = (Long) e.getProperty(TOTAL_LINE_COUNT); - } catch (ClassCastException exception) { - // Invalid contents or null values - logger.log(Level.WARNING, "Error parsing test entity.", exception); - return null; - } - return new TestCoverageStatusEntity(testName, timestamp, coveredLineCount, totalLineCount); + /** + * TestCoverageStatusEntity updatedTotalLineCount field + */ + @Index + @Getter + @Setter + long updatedTotalLineCount; + + /** + * TestCoverageStatusEntity updatedDate field + */ + @Index + @Getter + @Setter + Date updatedDate; + + /** + * Create a TestCoverageStatusEntity object with status metadata. + * + * @param testName The name of the test. + * @param timestamp The timestamp indicating the most recent test run event in the test state. + * @param coveredLineCount The number of lines covered. + * @param totalLineCount The total number of lines. + */ + public TestCoverageStatusEntity( + String testName, long timestamp, long coveredLineCount, long totalLineCount) { + this.testName = testName; + this.updatedTimestamp = timestamp; + this.coveredLineCount = coveredLineCount; + this.totalLineCount = totalLineCount; + } + + /** + * find TestCoverageStatus entity by ID + */ + public static TestCoverageStatusEntity findById(String testName) { + return ofy().load().type(TestCoverageStatusEntity.class).id(testName).now(); + } + + /** + * Get all TestCoverageStatusEntity List + */ + public static Map<String, TestCoverageStatusEntity> getTestCoverageStatusMap() { + List<TestCoverageStatusEntity> testCoverageStatusEntityList = getAllTestCoverage(); + + Map<String, TestCoverageStatusEntity> testCoverageStatusMap = testCoverageStatusEntityList + .stream() + .collect( + Collectors.toMap(t -> t.getTestName(), t -> t) + ); + return testCoverageStatusMap; + } + + /** + * Get all TestCoverageStatusEntity List + */ + public static List<TestCoverageStatusEntity> getAllTestCoverage() { + return ofy().load().type(TestCoverageStatusEntity.class).list(); + } + + /** + * Saving function for the instance of this class + */ + public void save() { + this.updatedDate = new Date(); + ofy().save().entity(this).now(); + } + + public Entity toEntity() { + Entity testEntity = new Entity(KIND, this.testName); + testEntity.setProperty(UPDATED_TIMESTAMP, this.updatedTimestamp); + testEntity.setProperty(COVERED_LINE_COUNT, this.coveredLineCount); + testEntity.setProperty(TOTAL_LINE_COUNT, this.totalLineCount); + return testEntity; + } + + /** + * Convert an Entity object to a TestCoverageStatusEntity. + * + * @param e The entity to process. + * @return TestCoverageStatusEntity object with the properties from e processed, or null if + * incompatible. + */ + @SuppressWarnings("unchecked") + public static TestCoverageStatusEntity fromEntity(Entity e) { + if (!e.getKind().equals(KIND) + || e.getKey().getName() == null + || !e.hasProperty(UPDATED_TIMESTAMP) + || !e.hasProperty(COVERED_LINE_COUNT) + || !e.hasProperty(TOTAL_LINE_COUNT)) { + logger.log(Level.WARNING, "Missing test attributes in entity: " + e.toString()); + return null; + } + String testName = e.getKey().getName(); + long timestamp = 0; + long coveredLineCount = -1; + long totalLineCount = -1; + try { + timestamp = (long) e.getProperty(UPDATED_TIMESTAMP); + coveredLineCount = (Long) e.getProperty(COVERED_LINE_COUNT); + totalLineCount = (Long) e.getProperty(TOTAL_LINE_COUNT); + } catch (ClassCastException exception) { + // Invalid contents or null values + logger.log(Level.WARNING, "Error parsing test entity.", exception); + return null; } + return new TestCoverageStatusEntity(testName, timestamp, coveredLineCount, totalLineCount); + } } diff --git a/src/main/java/com/android/vts/entity/TestEntity.java b/src/main/java/com/android/vts/entity/TestEntity.java index 9a8d32a..e48d759 100644 --- a/src/main/java/com/android/vts/entity/TestEntity.java +++ b/src/main/java/com/android/vts/entity/TestEntity.java @@ -16,22 +16,44 @@ package com.android.vts.entity; -import com.google.appengine.api.datastore.Entity; +import static com.googlecode.objectify.ObjectifyService.ofy; + import com.google.appengine.api.datastore.Key; import com.google.appengine.api.datastore.KeyFactory; +import com.googlecode.objectify.annotation.Cache; +import com.googlecode.objectify.annotation.Entity; +import com.googlecode.objectify.annotation.Id; +import com.googlecode.objectify.annotation.Index; +import java.io.Serializable; +import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +@Entity(name="Test") +@Cache +@Data +@NoArgsConstructor /** Entity describing test metadata. */ -public class TestEntity implements DashboardEntity { +public class TestEntity implements Serializable { protected static final Logger logger = Logger.getLogger(TestEntity.class.getName()); public static final String KIND = "Test"; public static final String HAS_PROFILING_DATA = "hasProfilingData"; - public final String testName; - public final Key key; - public boolean hasProfilingData; + @Id + @Getter + @Setter + private String testName; + + @Index + @Getter + @Setter + private boolean hasProfilingData; /** * Create a TestEntity object. @@ -41,7 +63,6 @@ public class TestEntity implements DashboardEntity { */ public TestEntity(String testName, boolean hasProfilingData) { this.testName = testName; - this.key = KeyFactory.createKey(KIND, testName); this.hasProfilingData = hasProfilingData; } @@ -54,20 +75,29 @@ public class TestEntity implements DashboardEntity { this(testName, false); } - @Override - public Entity toEntity() { - Entity testEntity = new Entity(this.key); + public com.google.appengine.api.datastore.Entity toEntity() { + com.google.appengine.api.datastore.Entity testEntity = new com.google.appengine.api.datastore.Entity(this.getOldKey()); testEntity.setProperty(HAS_PROFILING_DATA, this.hasProfilingData); return testEntity; } /** - * Set to true if the test has profiling data. - * - * @param hasProfilingData The value to store. + * Get key info from appengine based library. */ - public void setHasProfilingData(boolean hasProfilingData) { - this.hasProfilingData = hasProfilingData; + public Key getOldKey() { + return KeyFactory.createKey(KIND, testName); + } + + public static List<String> getAllTestNames() { + List<TestEntity> testEntityList = getAllTest(); + + List<String> allTestNames = testEntityList.stream() + .map(te -> te.getTestName()).collect(Collectors.toList()); + return allTestNames; + } + + public static List<TestEntity> getAllTest() { + return ofy().load().type(TestEntity.class).list(); } @Override @@ -88,7 +118,7 @@ public class TestEntity implements DashboardEntity { * @return TestEntity object with the properties from e processed, or null if incompatible. */ @SuppressWarnings("unchecked") - public static TestEntity fromEntity(Entity e) { + public static TestEntity fromEntity(com.google.appengine.api.datastore.Entity e) { if (!e.getKind().equals(KIND) || e.getKey().getName() == null) { logger.log(Level.WARNING, "Missing test attributes in entity: " + e.toString()); return null; @@ -100,4 +130,10 @@ public class TestEntity implements DashboardEntity { } return new TestEntity(testName, hasProfilingData); } + + /** Saving function for the instance of this class */ + public void save() { + ofy().save().entity(this).now(); + } + } diff --git a/src/main/java/com/android/vts/entity/TestPlanEntity.java b/src/main/java/com/android/vts/entity/TestPlanEntity.java index 82d7e5e..805c3cd 100644 --- a/src/main/java/com/android/vts/entity/TestPlanEntity.java +++ b/src/main/java/com/android/vts/entity/TestPlanEntity.java @@ -16,12 +16,24 @@ package com.android.vts.entity; +import static com.googlecode.objectify.ObjectifyService.ofy; + import com.google.appengine.api.datastore.Entity; +import com.googlecode.objectify.annotation.Cache; +import com.googlecode.objectify.annotation.Id; +import java.io.Serializable; +import java.util.Date; import java.util.logging.Level; import java.util.logging.Logger; +import lombok.Data; +import lombok.NoArgsConstructor; +@com.googlecode.objectify.annotation.Entity(name="TestPlan") +@Cache +@Data +@NoArgsConstructor /** Entity describing test plan metadata. */ -public class TestPlanEntity implements DashboardEntity { +public class TestPlanEntity implements Serializable { protected static final Logger logger = Logger.getLogger(TestPlanEntity.class.getName()); public static final String KIND = "TestPlan"; @@ -29,7 +41,8 @@ public class TestPlanEntity implements DashboardEntity { // Property keys public static final String TEST_PLAN_NAME = "testPlanName"; - public final String testPlanName; + @Id + public String testPlanName; /** * Create a TestPlanEntity object. @@ -40,7 +53,6 @@ public class TestPlanEntity implements DashboardEntity { this.testPlanName = testPlanName; } - @Override public Entity toEntity() { Entity planEntity = new Entity(KIND, this.testPlanName); return planEntity; @@ -62,4 +74,9 @@ public class TestPlanEntity implements DashboardEntity { String testPlanName = e.getKey().getName(); return new TestPlanEntity(testPlanName); } + + /** Saving function for the instance of this class */ + public void save() { + ofy().save().entity(this).now(); + } } diff --git a/src/main/java/com/android/vts/entity/TestPlanRunEntity.java b/src/main/java/com/android/vts/entity/TestPlanRunEntity.java index cc161ac..e72e454 100644 --- a/src/main/java/com/android/vts/entity/TestPlanRunEntity.java +++ b/src/main/java/com/android/vts/entity/TestPlanRunEntity.java @@ -22,120 +22,197 @@ import com.google.appengine.api.datastore.Key; import com.google.appengine.api.datastore.KeyFactory; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; +import com.googlecode.objectify.annotation.Cache; +import com.googlecode.objectify.annotation.Id; +import com.googlecode.objectify.annotation.Ignore; +import com.googlecode.objectify.annotation.Index; +import com.googlecode.objectify.annotation.Parent; +import java.io.Serializable; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +@com.googlecode.objectify.annotation.Entity(name = "TestPlanRun") +@Cache +@Data +@NoArgsConstructor /** Entity describing test plan run information. */ -public class TestPlanRunEntity implements DashboardEntity { - protected static final Logger logger = Logger.getLogger(TestPlanRunEntity.class.getName()); - - public static final String KIND = "TestPlanRun"; - - // Property keys - public static final String TEST_PLAN_NAME = "testPlanName"; - public static final String TYPE = "type"; - public static final String START_TIMESTAMP = "startTimestamp"; - public static final String END_TIMESTAMP = "endTimestamp"; - public static final String TEST_BUILD_ID = "testBuildId"; - public static final String PASS_COUNT = "passCount"; - public static final String FAIL_COUNT = "failCount"; - public static final String TEST_RUNS = "testRuns"; - - public final Key key; - public final String testPlanName; - public final TestRunType type; - public final long startTimestamp; - public final long endTimestamp; - public final String testBuildId; - public final long passCount; - public final long failCount; - public final List<Key> testRuns; - - /** - * Create a TestPlanRunEntity object describing a test plan run. - * - * @param parentKey The key for the parent entity in the database. - * @param type The test run type (e.g. presubmit, postsubmit, other) - * @param startTimestamp The time in microseconds when the test plan run started. - * @param endTimestamp The time in microseconds when the test plan run ended. - * @param testBuildId The build ID of the VTS test build. - * @param passCount The number of passing test cases in the run. - * @param failCount The number of failing test cases in the run. - * @param testRuns A list of keys to the TestRunEntity objects for the plan run run. - */ - public TestPlanRunEntity(Key parentKey, String testPlanName, TestRunType type, - long startTimestamp, long endTimestamp, String testBuildId, long passCount, - long failCount, List<Key> testRuns) { - this.key = KeyFactory.createKey(parentKey, KIND, startTimestamp); - this.testPlanName = testPlanName; - this.type = type; - this.startTimestamp = startTimestamp; - this.endTimestamp = endTimestamp; - this.testBuildId = testBuildId; - this.passCount = passCount; - this.failCount = failCount; - this.testRuns = testRuns; - } +public class TestPlanRunEntity implements Serializable { - @Override - public Entity toEntity() { - Entity planRun = new Entity(this.key); - planRun.setProperty(TEST_PLAN_NAME, this.testPlanName); - planRun.setProperty(TYPE, this.type.getNumber()); - planRun.setProperty(START_TIMESTAMP, this.startTimestamp); - planRun.setProperty(END_TIMESTAMP, this.endTimestamp); - planRun.setProperty(TEST_BUILD_ID, this.testBuildId.toLowerCase()); - planRun.setProperty(PASS_COUNT, this.passCount); - planRun.setProperty(FAIL_COUNT, this.failCount); - if (this.testRuns != null && this.testRuns.size() > 0) { - planRun.setUnindexedProperty(TEST_RUNS, this.testRuns); - } - return planRun; - } + protected static final Logger logger = Logger.getLogger(TestPlanRunEntity.class.getName()); + + public static final String KIND = "TestPlanRun"; + + // Property keys + public static final String TEST_PLAN_NAME = "testPlanName"; + public static final String TYPE = "type"; + public static final String START_TIMESTAMP = "startTimestamp"; + public static final String END_TIMESTAMP = "endTimestamp"; + public static final String TEST_BUILD_ID = "testBuildId"; + public static final String PASS_COUNT = "passCount"; + public static final String FAIL_COUNT = "failCount"; + public static final String TEST_RUNS = "testRuns"; + + @Ignore + public Key key; + + @Id + @Getter + @Setter + private Long ID; + + @Parent + @Getter + @Setter + private com.googlecode.objectify.Key<?> testParent; + + @Index + @Getter + @Setter + private String testPlanName; + + @Index + @Getter + @Setter + private TestRunType type; + + @Index + @Getter + @Setter + private long startTimestamp; + + @Index + @Getter + @Setter + private long endTimestamp; - /** - * Convert an Entity object to a TestPlanRunEntity. - * - * @param e The entity to process. - * @return TestPlanRunEntity object with the properties from e processed, or null if - * incompatible. - */ - @SuppressWarnings("unchecked") - public static TestPlanRunEntity fromEntity(Entity e) { - if (!e.getKind().equals(KIND) || !e.hasProperty(TEST_PLAN_NAME) || !e.hasProperty(TYPE) - || !e.hasProperty(START_TIMESTAMP) || !e.hasProperty(END_TIMESTAMP) - || !e.hasProperty(TEST_BUILD_ID) || !e.hasProperty(PASS_COUNT) - || !e.hasProperty(FAIL_COUNT) || !e.hasProperty(TEST_RUNS)) { - logger.log(Level.WARNING, "Missing test run attributes in entity: " + e.toString()); - return null; - } - try { - String testPlanName = (String) e.getProperty(TEST_PLAN_NAME); - TestRunType type = TestRunType.fromNumber((int) (long) e.getProperty(TYPE)); - long startTimestamp = (long) e.getProperty(START_TIMESTAMP); - long endTimestamp = (long) e.getProperty(END_TIMESTAMP); - String testBuildId = (String) e.getProperty(TEST_BUILD_ID); - long passCount = (long) e.getProperty(PASS_COUNT); - long failCount = (long) e.getProperty(FAIL_COUNT); - List<Key> testRuns = (List<Key>) e.getProperty(TEST_RUNS); - return new TestPlanRunEntity(e.getKey().getParent(), testPlanName, type, startTimestamp, - endTimestamp, testBuildId, passCount, failCount, testRuns); - } catch (ClassCastException exception) { - // Invalid cast - logger.log(Level.WARNING, "Error parsing test plan run entity.", exception); - } - return null; + @Index + @Getter + @Setter + private String testBuildId; + + @Index + @Getter + @Setter + private long passCount; + + @Index + @Getter + @Setter + private long failCount; + + @Getter + @Setter + private List<Key> testRuns; + + @Ignore + @Getter + @Setter + private List<com.googlecode.objectify.Key<?>> ofyTestRuns; + + /** + * Create a TestPlanRunEntity object describing a test plan run. + * + * @param parentKey The key for the parent entity in the database. + * @param type The test run type (e.g. presubmit, postsubmit, other) + * @param startTimestamp The time in microseconds when the test plan run started. + * @param endTimestamp The time in microseconds when the test plan run ended. + * @param testBuildId The build ID of the VTS test build. + * @param passCount The number of passing test cases in the run. + * @param failCount The number of failing test cases in the run. + * @param testRuns A list of keys to the TestRunEntity objects for the plan run run. + */ + public TestPlanRunEntity(Key parentKey, String testPlanName, TestRunType type, + long startTimestamp, long endTimestamp, String testBuildId, long passCount, + long failCount, List<Key> testRuns) { + this.key = KeyFactory.createKey(parentKey, KIND, startTimestamp); + this.testPlanName = testPlanName; + this.type = type; + this.startTimestamp = startTimestamp; + this.endTimestamp = endTimestamp; + this.testBuildId = testBuildId; + this.passCount = passCount; + this.failCount = failCount; + this.testRuns = testRuns; + this.ofyTestRuns = testRuns.stream().map(testRun -> { + com.googlecode.objectify.Key testParentKey = com.googlecode.objectify.Key + .create(TestEntity.class, testRun.getParent().getName()); + return com.googlecode.objectify.Key + .create(testParentKey, TestRunEntity.class, testRun.getId()); + }).collect(Collectors.toList()); + + } + + public Entity toEntity() { + Entity planRun = new Entity(this.key); + planRun.setProperty(TEST_PLAN_NAME, this.testPlanName); + planRun.setProperty(TYPE, this.type.getNumber()); + planRun.setProperty(START_TIMESTAMP, this.startTimestamp); + planRun.setProperty(END_TIMESTAMP, this.endTimestamp); + planRun.setProperty(TEST_BUILD_ID, this.testBuildId.toLowerCase()); + planRun.setProperty(PASS_COUNT, this.passCount); + planRun.setProperty(FAIL_COUNT, this.failCount); + if (this.testRuns != null && this.testRuns.size() > 0) { + planRun.setUnindexedProperty(TEST_RUNS, this.testRuns); } + return planRun; + } + + /** + * Get key info from appengine based library. + * + * @param parentKey parent key. + */ + public Key getOldKey(Key parentKey) { + return KeyFactory.createKey(parentKey, KIND, startTimestamp); + } - public JsonObject toJson() { - JsonObject json = new JsonObject(); - json.add(TEST_PLAN_NAME, new JsonPrimitive(this.testPlanName)); - json.add(TEST_BUILD_ID, new JsonPrimitive(this.testBuildId)); - json.add(PASS_COUNT, new JsonPrimitive(this.passCount)); - json.add(FAIL_COUNT, new JsonPrimitive(this.failCount)); - json.add(START_TIMESTAMP, new JsonPrimitive(this.startTimestamp)); - json.add(END_TIMESTAMP, new JsonPrimitive(this.endTimestamp)); - return json; + /** + * Convert an Entity object to a TestPlanRunEntity. + * + * @param e The entity to process. + * @return TestPlanRunEntity object with the properties from e processed, or null if incompatible. + */ + @SuppressWarnings("unchecked") + public static TestPlanRunEntity fromEntity(Entity e) { + if (!e.getKind().equals(KIND) || !e.hasProperty(TEST_PLAN_NAME) || !e.hasProperty(TYPE) + || !e.hasProperty(START_TIMESTAMP) || !e.hasProperty(END_TIMESTAMP) + || !e.hasProperty(TEST_BUILD_ID) || !e.hasProperty(PASS_COUNT) + || !e.hasProperty(FAIL_COUNT) || !e.hasProperty(TEST_RUNS)) { + logger.log(Level.WARNING, "Missing test run attributes in entity: " + e.toString()); + return null; } + try { + String testPlanName = (String) e.getProperty(TEST_PLAN_NAME); + TestRunType type = TestRunType.fromNumber((int) (long) e.getProperty(TYPE)); + long startTimestamp = (long) e.getProperty(START_TIMESTAMP); + long endTimestamp = (long) e.getProperty(END_TIMESTAMP); + String testBuildId = (String) e.getProperty(TEST_BUILD_ID); + long passCount = (long) e.getProperty(PASS_COUNT); + long failCount = (long) e.getProperty(FAIL_COUNT); + List<Key> testRuns = (List<Key>) e.getProperty(TEST_RUNS); + return new TestPlanRunEntity(e.getKey().getParent(), testPlanName, type, startTimestamp, + endTimestamp, testBuildId, passCount, failCount, testRuns); + } catch (ClassCastException exception) { + // Invalid cast + logger.log(Level.WARNING, "Error parsing test plan run entity.", exception); + } + return null; + } + + public JsonObject toJson() { + JsonObject json = new JsonObject(); + json.add(TEST_PLAN_NAME, new JsonPrimitive(this.testPlanName)); + json.add(TEST_BUILD_ID, new JsonPrimitive(this.testBuildId)); + json.add(PASS_COUNT, new JsonPrimitive(this.passCount)); + json.add(FAIL_COUNT, new JsonPrimitive(this.failCount)); + json.add(START_TIMESTAMP, new JsonPrimitive(this.startTimestamp)); + json.add(END_TIMESTAMP, new JsonPrimitive(this.endTimestamp)); + return json; + } } diff --git a/src/main/java/com/android/vts/entity/TestRunEntity.java b/src/main/java/com/android/vts/entity/TestRunEntity.java index b207499..1766363 100644 --- a/src/main/java/com/android/vts/entity/TestRunEntity.java +++ b/src/main/java/com/android/vts/entity/TestRunEntity.java @@ -25,13 +25,27 @@ import com.google.gson.Gson; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonPrimitive; +import com.googlecode.objectify.annotation.Cache; +import com.googlecode.objectify.annotation.Id; +import com.googlecode.objectify.annotation.Ignore; +import com.googlecode.objectify.annotation.Index; +import com.googlecode.objectify.annotation.Parent; +import java.io.Serializable; import java.util.ArrayList; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +@com.googlecode.objectify.annotation.Entity(name="TestRun") +@Cache +@Data +@NoArgsConstructor /** Entity describing test run information. */ -public class TestRunEntity implements DashboardEntity { +public class TestRunEntity implements Serializable { protected static final Logger logger = Logger.getLogger(TestRunEntity.class.getName()); /** Enum for classifying test run types. */ @@ -116,19 +130,81 @@ public class TestRunEntity implements DashboardEntity { public static final String TOTAL_LINE_COUNT = "totalLineCount"; public static final String COVERED_LINE_COUNT = "coveredLineCount"; - public final Key key; - public final TestRunType type; - public final long startTimestamp; - public final long endTimestamp; - public final String testBuildId; - public final String hostName; - public final long passCount; - public final long failCount; - public final boolean hasCoverage; - public final long coveredLineCount; - public final long totalLineCount; - public final List<Long> testCaseIds; - public final List<String> links; + @Ignore + private Key key; + + @Id + @Getter + @Setter + private Long ID; + + @Parent + @Getter + @Setter + private com.googlecode.objectify.Key<?> testParent; + + @Index + @Getter + @Setter + private TestRunType type; + + @Index + @Getter + @Setter + private long startTimestamp; + + @Index + @Getter + @Setter + private long endTimestamp; + + @Index + @Getter + @Setter + private String testBuildId; + + @Index + @Getter + @Setter + private String testName; + + @Index + @Getter + @Setter + private String hostName; + + @Index + @Getter + @Setter + private long passCount; + + @Index + @Getter + @Setter + private long failCount; + + @Index + @Getter + @Setter + private boolean hasCoverage; + + @Index + @Getter + @Setter + private long coveredLineCount; + + @Index + @Getter + @Setter + private long totalLineCount; + + @Getter + @Setter + private List<Long> testCaseIds; + + @Getter + @Setter + private List<String> links; /** * Create a TestRunEntity object describing a test run. @@ -186,7 +262,6 @@ public class TestRunEntity implements DashboardEntity { failCount, testCaseIds, links, 0, 0); } - @Override public Entity toEntity() { Entity testRunEntity = new Entity(this.key); testRunEntity.setProperty(TEST_NAME, this.key.getParent().getName()); @@ -211,6 +286,15 @@ public class TestRunEntity implements DashboardEntity { } /** + * Get key info from appengine based library. + * + * @param parentKey parent key. + */ + public Key getOldKey(Key parentKey) { + return KeyFactory.createKey(parentKey, KIND, startTimestamp); + } + + /** * Convert an Entity object to a TestRunEntity. * * @param e The entity to process. diff --git a/src/main/java/com/android/vts/entity/UserEntity.java b/src/main/java/com/android/vts/entity/UserEntity.java index e5f9f8f..66216af 100644 --- a/src/main/java/com/android/vts/entity/UserEntity.java +++ b/src/main/java/com/android/vts/entity/UserEntity.java @@ -17,10 +17,13 @@ package com.android.vts.entity; import com.googlecode.objectify.Key; +import com.googlecode.objectify.Ref; import com.googlecode.objectify.annotation.Cache; import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Id; import com.googlecode.objectify.annotation.Index; +import com.googlecode.objectify.annotation.Load; +import java.util.ArrayList; import java.util.List; import lombok.EqualsAndHashCode; import lombok.Getter; @@ -33,7 +36,7 @@ import static com.googlecode.objectify.ObjectifyService.ofy; /** Entity Class for User */ @Cache @Entity -@EqualsAndHashCode(of = "id") +@EqualsAndHashCode(of = "email") @NoArgsConstructor public class UserEntity { @@ -55,16 +58,24 @@ public class UserEntity { /** When this record was created or updated */ @Index @Getter Date updated; - /** Construction function for UserEntity Class */ + @Load + @Getter + List<Ref<RoleEntity>> roles; + + /** Constructor function for UserEntity Class */ public UserEntity( String email, String name, - String company) { + String company, + String roleName) { + RoleEntity role = ofy().load().type(RoleEntity.class).id(roleName).now(); + this.email = email; this.name = name; this.enable = true; this.isAdmin = false; this.company = company; + this.roles.add(Ref.create(role)); } /** Saving function for the instance of this class */ @@ -73,16 +84,19 @@ public class UserEntity { ofy().save().entity(this).now(); } - public static List<UserEntity> getAdminUserList(String adminEmail) { + /** Get admin user list by admin email */ + public static UserEntity getAdminUser(String adminEmail) { Key key = Key.create(UserEntity.class, adminEmail); return ofy().load() .type(UserEntity.class) .filterKey(key) .filter("enable", true) .filter("isAdmin", true) - .list(); + .first() + .now(); } + /** Get all admin user list */ public static List<UserEntity> getAdminUserList() { return ofy().load() .type(UserEntity.class) @@ -91,6 +105,7 @@ public class UserEntity { .list(); } + /** Get normal user list */ public static List<UserEntity> getUserList() { return ofy().load() .type(UserEntity.class) @@ -98,4 +113,15 @@ public class UserEntity { .filter("isAdmin", false) .list(); } + + /** Get user by email */ + public static UserEntity getUser(String email) { + Key key = Key.create(UserEntity.class, email); + return ofy().load() + .type(UserEntity.class) + .filterKey(key) + .filter("enable", true) + .first() + .now(); + } } diff --git a/src/main/java/com/android/vts/job/VtsAlertJobServlet.java b/src/main/java/com/android/vts/job/VtsAlertJobServlet.java index c4aa4a5..511017f 100644 --- a/src/main/java/com/android/vts/job/VtsAlertJobServlet.java +++ b/src/main/java/com/android/vts/job/VtsAlertJobServlet.java @@ -253,7 +253,7 @@ public class VtsAlertJobServlet extends HttpServlet { mostRecentRun = testRun; } List<Key> testCaseKeys = new ArrayList<>(); - for (long testCaseId : testRun.testCaseIds) { + for (long testCaseId : testRun.getTestCaseIds()) { testCaseKeys.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); } Map<Key, Entity> entityMap = datastore.get(testCaseKeys); @@ -302,7 +302,7 @@ public class VtsAlertJobServlet extends HttpServlet { Set<String> buildIdList = new HashSet<>(); List<DeviceInfoEntity> devices = new ArrayList<>(); - Query deviceQuery = new Query(DeviceInfoEntity.KIND).setAncestor(mostRecentRun.key); + Query deviceQuery = new Query(DeviceInfoEntity.KIND).setAncestor(mostRecentRun.getKey()); for (Entity device : datastore.prepare(deviceQuery).asIterable()) { DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device); if (deviceEntity == null) { @@ -411,8 +411,8 @@ public class VtsAlertJobServlet extends HttpServlet { } } - String testName = mostRecentRun.key.getParent().getName(); - String uploadDateString = TimeUtil.getDateString(mostRecentRun.startTimestamp); + String testName = mostRecentRun.getKey().getParent().getName(); + String uploadDateString = TimeUtil.getDateString(mostRecentRun.getStartTimestamp()); String subject = "VTS Test Alert: " + testName + " @ " + uploadDateString; if (newTestcaseFailures.size() > 0) { String body = @@ -474,7 +474,7 @@ public class VtsAlertJobServlet extends HttpServlet { } return new TestStatusEntity( testName, - mostRecentRun.startTimestamp, + mostRecentRun.getStartTimestamp(), passingTestcaseCount, failingTestCases.size(), failingTestCases); diff --git a/src/main/java/com/android/vts/job/VtsCoverageAlertJobServlet.java b/src/main/java/com/android/vts/job/VtsCoverageAlertJobServlet.java index 01beba2..a112496 100644 --- a/src/main/java/com/android/vts/job/VtsCoverageAlertJobServlet.java +++ b/src/main/java/com/android/vts/job/VtsCoverageAlertJobServlet.java @@ -16,6 +16,8 @@ package com.android.vts.job; +import static com.googlecode.objectify.ObjectifyService.ofy; + import com.android.vts.entity.DeviceInfoEntity; import com.android.vts.entity.TestCoverageStatusEntity; import com.android.vts.entity.TestRunEntity; @@ -52,304 +54,275 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import org.apache.commons.lang.StringUtils; -/** Coverage notification job. */ +/** + * Coverage notification job. + */ public class VtsCoverageAlertJobServlet extends HttpServlet { - private static final String COVERAGE_ALERT_URL = "/task/vts_coverage_job"; - protected static final Logger logger = - Logger.getLogger(VtsCoverageAlertJobServlet.class.getName()); - protected static final double CHANGE_ALERT_THRESHOLD = 0.05; - protected static final double GOOD_THRESHOLD = 0.7; - protected static final double BAD_THRESHOLD = 0.3; - protected static final DecimalFormat FORMATTER; + private static final String COVERAGE_ALERT_URL = "/task/vts_coverage_job"; + protected static final Logger logger = + Logger.getLogger(VtsCoverageAlertJobServlet.class.getName()); + protected static final double CHANGE_ALERT_THRESHOLD = 0.05; + protected static final double GOOD_THRESHOLD = 0.7; + protected static final double BAD_THRESHOLD = 0.3; - /** Initialize the decimal formatter. */ - static { - FORMATTER = new DecimalFormat("#.#"); - FORMATTER.setRoundingMode(RoundingMode.HALF_UP); - } + protected static final DecimalFormat FORMATTER; - /** - * Gets a new coverage status and adds notification emails to the messages list. - * - * Send an email to notify subscribers in the event that a test goes up or down by more than - * 5%, becomes higher or lower than 70%, or becomes higher or lower than 30%. - * - * @param status The TestCoverageStatusEntity object for the test. - * @param testRunKey The key for TestRunEntity whose data to process and reflect in the state. - * @param link The string URL linking to the test's status table. - * @param emailAddresses The list of email addresses to send notifications to. - * @param messages The email Message queue. - * @returns TestCoverageStatusEntity or null if no update is available. - * @throws IOException - */ - public static TestCoverageStatusEntity getTestCoverageStatus( - TestCoverageStatusEntity status, - Key testRunKey, - String link, - List<String> emailAddresses, - List<Message> messages) - throws IOException { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); + /** Initialize the decimal formatter. */ + static { + FORMATTER = new DecimalFormat("#.#"); + FORMATTER.setRoundingMode(RoundingMode.HALF_UP); + } - String testName = status.testName; + /** + * Gets a new coverage status and adds notification emails to the messages list. + * + * Send an email to notify subscribers in the event that a test goes up or down by more than 5%, + * becomes higher or lower than 70%, or becomes higher or lower than 30%. + * + * @param status The TestCoverageStatusEntity object for the test. + * @param testRunKey The key for TestRunEntity whose data to process and reflect in the state. + * @param link The string URL linking to the test's status table. + * @param emailAddresses The list of email addresses to send notifications to. + * @param messages The email Message queue. + * @returns TestCoverageStatusEntity or null if no update is available. + */ + public static TestCoverageStatusEntity getTestCoverageStatus( + TestCoverageStatusEntity status, + Key testRunKey, + String link, + List<String> emailAddresses, + List<Message> messages) + throws IOException { + DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - double previousPct; - double coveragePct; - if (status == null || status.totalLineCount <= 0 || status.coveredLineCount < 0) { - previousPct = 0; - } else { - previousPct = ((double) status.coveredLineCount) / status.totalLineCount; - } + String testName = status.getTestName(); - Entity testRun; - try { - testRun = datastore.get(testRunKey); - } catch (EntityNotFoundException e) { - logger.log(Level.WARNING, "Test run not found: " + testRunKey); - return null; - } + double previousPct; + double coveragePct; + if (status == null || status.getTotalLineCount() <= 0 || status.getCoveredLineCount() < 0) { + previousPct = 0; + } else { + previousPct = ((double) status.getCoveredLineCount()) / status.getTotalLineCount(); + } - TestRunEntity testRunEntity = TestRunEntity.fromEntity(testRun); - if (testRunEntity == null || !testRunEntity.hasCoverage) { - return null; - } - if (testRunEntity.totalLineCount <= 0 || testRunEntity.coveredLineCount < 0) { - coveragePct = 0; - } else { - coveragePct = ((double) testRunEntity.coveredLineCount) / testRunEntity.totalLineCount; - } + Entity testRun; + try { + testRun = datastore.get(testRunKey); + } catch (EntityNotFoundException e) { + logger.log(Level.WARNING, "Test run not found: " + testRunKey); + return null; + } - Set<String> buildIdList = new HashSet<>(); - Query deviceQuery = new Query(DeviceInfoEntity.KIND).setAncestor(testRun.getKey()); - List<DeviceInfoEntity> devices = new ArrayList<>(); - for (Entity device : datastore.prepare(deviceQuery).asIterable()) { - DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device); - if (deviceEntity == null) { - continue; - } - devices.add(deviceEntity); - buildIdList.add(deviceEntity.buildId); - } - String deviceBuild = StringUtils.join(buildIdList, ", "); - String footer = EmailHelper.getEmailFooter(testRunEntity, devices, link); + TestRunEntity testRunEntity = TestRunEntity.fromEntity(testRun); + if (testRunEntity == null || !testRunEntity.isHasCoverage()) { + return null; + } + if (testRunEntity.getTotalLineCount() <= 0 || testRunEntity.getCoveredLineCount() < 0) { + coveragePct = 0; + } else { + coveragePct = + ((double) testRunEntity.getCoveredLineCount()) / testRunEntity.getTotalLineCount(); + } - String subject = null; - String body = null; - String subjectSuffix = " @ " + deviceBuild; - if (coveragePct >= GOOD_THRESHOLD && previousPct < GOOD_THRESHOLD) { - // Coverage entered the good zone - subject = - "Congratulations! " - + testName - + " has exceeded " - + FORMATTER.format(GOOD_THRESHOLD * 100) - + "% coverage" - + subjectSuffix; - body = - "Hello,<br><br>The " - + testName - + " has achieved " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } else if (coveragePct < GOOD_THRESHOLD && previousPct >= GOOD_THRESHOLD) { - // Coverage dropped out of the good zone - subject = - "Warning! " - + testName - + " has dropped below " - + FORMATTER.format(GOOD_THRESHOLD * 100) - + "% coverage" - + subjectSuffix; - ; - body = - "Hello,<br><br>The test " - + testName - + " has dropped to " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } else if (coveragePct <= BAD_THRESHOLD && previousPct > BAD_THRESHOLD) { - // Coverage entered into the bad zone - subject = - "Warning! " - + testName - + " has dropped below " - + FORMATTER.format(BAD_THRESHOLD * 100) - + "% coverage" - + subjectSuffix; - body = - "Hello,<br><br>The test " - + testName - + " has dropped to " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } else if (coveragePct > BAD_THRESHOLD && previousPct <= BAD_THRESHOLD) { - // Coverage emerged from the bad zone - subject = - "Congratulations! " - + testName - + " has exceeded " - + FORMATTER.format(BAD_THRESHOLD * 100) - + "% coverage" - + subjectSuffix; - body = - "Hello,<br><br>The test " - + testName - + " has achived " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } else if (coveragePct - previousPct < -CHANGE_ALERT_THRESHOLD) { - // Send a coverage drop alert - subject = - "Warning! " - + testName - + "'s code coverage has decreased by more than " - + FORMATTER.format(CHANGE_ALERT_THRESHOLD * 100) - + "%" - + subjectSuffix; - body = - "Hello,<br><br>The test " - + testName - + " has dropped from " - + FORMATTER.format(previousPct * 100) - + "% code coverage to " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } else if (coveragePct - previousPct > CHANGE_ALERT_THRESHOLD) { - // Send a coverage improvement alert - subject = - testName - + "'s code coverage has increased by more than " - + FORMATTER.format(CHANGE_ALERT_THRESHOLD * 100) - + "%" - + subjectSuffix; - body = - "Hello,<br><br>The test " - + testName - + " has increased from " - + FORMATTER.format(previousPct * 100) - + "% code coverage to " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } - if (subject != null && body != null) { - try { - messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); - } catch (MessagingException | UnsupportedEncodingException e) { - logger.log(Level.WARNING, "Error composing email : ", e); - } - } - return new TestCoverageStatusEntity( - testName, - testRunEntity.startTimestamp, - testRunEntity.coveredLineCount, - testRunEntity.totalLineCount); + Set<String> buildIdList = new HashSet<>(); + Query deviceQuery = new Query(DeviceInfoEntity.KIND).setAncestor(testRun.getKey()); + List<DeviceInfoEntity> devices = new ArrayList<>(); + for (Entity device : datastore.prepare(deviceQuery).asIterable()) { + DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device); + if (deviceEntity == null) { + continue; + } + devices.add(deviceEntity); + buildIdList.add(deviceEntity.buildId); } + String deviceBuild = StringUtils.join(buildIdList, ", "); + String footer = EmailHelper.getEmailFooter(testRunEntity, devices, link); - /** - * Add a task to process coverage data - * - * @param testRunKey The key of the test run whose data process. - */ - public static void addTask(Key testRunKey) { - Queue queue = QueueFactory.getDefaultQueue(); - String keyString = KeyFactory.keyToString(testRunKey); - queue.add( - TaskOptions.Builder.withUrl(COVERAGE_ALERT_URL) - .param("runKey", keyString) - .method(TaskOptions.Method.POST)); + String subject = null; + String body = null; + String subjectSuffix = " @ " + deviceBuild; + if (coveragePct >= GOOD_THRESHOLD && previousPct < GOOD_THRESHOLD) { + // Coverage entered the good zone + subject = + "Congratulations! " + + testName + + " has exceeded " + + FORMATTER.format(GOOD_THRESHOLD * 100) + + "% coverage" + + subjectSuffix; + body = + "Hello,<br><br>The " + + testName + + " has achieved " + + FORMATTER.format(coveragePct * 100) + + "% code coverage on device build ID(s): " + + deviceBuild + + "." + + footer; + } else if (coveragePct < GOOD_THRESHOLD && previousPct >= GOOD_THRESHOLD) { + // Coverage dropped out of the good zone + subject = + "Warning! " + + testName + + " has dropped below " + + FORMATTER.format(GOOD_THRESHOLD * 100) + + "% coverage" + + subjectSuffix; + ; + body = + "Hello,<br><br>The test " + + testName + + " has dropped to " + + FORMATTER.format(coveragePct * 100) + + "% code coverage on device build ID(s): " + + deviceBuild + + "." + + footer; + } else if (coveragePct <= BAD_THRESHOLD && previousPct > BAD_THRESHOLD) { + // Coverage entered into the bad zone + subject = + "Warning! " + + testName + + " has dropped below " + + FORMATTER.format(BAD_THRESHOLD * 100) + + "% coverage" + + subjectSuffix; + body = + "Hello,<br><br>The test " + + testName + + " has dropped to " + + FORMATTER.format(coveragePct * 100) + + "% code coverage on device build ID(s): " + + deviceBuild + + "." + + footer; + } else if (coveragePct > BAD_THRESHOLD && previousPct <= BAD_THRESHOLD) { + // Coverage emerged from the bad zone + subject = + "Congratulations! " + + testName + + " has exceeded " + + FORMATTER.format(BAD_THRESHOLD * 100) + + "% coverage" + + subjectSuffix; + body = + "Hello,<br><br>The test " + + testName + + " has achived " + + FORMATTER.format(coveragePct * 100) + + "% code coverage on device build ID(s): " + + deviceBuild + + "." + + footer; + } else if (coveragePct - previousPct < -CHANGE_ALERT_THRESHOLD) { + // Send a coverage drop alert + subject = + "Warning! " + + testName + + "'s code coverage has decreased by more than " + + FORMATTER.format(CHANGE_ALERT_THRESHOLD * 100) + + "%" + + subjectSuffix; + body = + "Hello,<br><br>The test " + + testName + + " has dropped from " + + FORMATTER.format(previousPct * 100) + + "% code coverage to " + + FORMATTER.format(coveragePct * 100) + + "% code coverage on device build ID(s): " + + deviceBuild + + "." + + footer; + } else if (coveragePct - previousPct > CHANGE_ALERT_THRESHOLD) { + // Send a coverage improvement alert + subject = + testName + + "'s code coverage has increased by more than " + + FORMATTER.format(CHANGE_ALERT_THRESHOLD * 100) + + "%" + + subjectSuffix; + body = + "Hello,<br><br>The test " + + testName + + " has increased from " + + FORMATTER.format(previousPct * 100) + + "% code coverage to " + + FORMATTER.format(coveragePct * 100) + + "% code coverage on device build ID(s): " + + deviceBuild + + "." + + footer; + } + if (subject != null && body != null) { + try { + messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); + } catch (MessagingException | UnsupportedEncodingException e) { + logger.log(Level.WARNING, "Error composing email : ", e); + } } + return new TestCoverageStatusEntity( + testName, + testRunEntity.getStartTimestamp(), + testRunEntity.getCoveredLineCount(), + testRunEntity.getTotalLineCount()); + } + + /** + * Add a task to process coverage data + * + * @param testRunKey The key of the test run whose data process. + */ + public static void addTask(Key testRunKey) { + Queue queue = QueueFactory.getDefaultQueue(); + String keyString = KeyFactory.keyToString(testRunKey); + queue.add( + TaskOptions.Builder.withUrl(COVERAGE_ALERT_URL) + .param("runKey", keyString) + .method(TaskOptions.Method.POST)); + } - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws IOException { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - String runKeyString = request.getParameter("runKey"); + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws IOException { + String runKeyString = request.getParameter("runKey"); - Key testRunKey; - try { - testRunKey = KeyFactory.stringToKey(runKeyString); - } catch (IllegalArgumentException e) { - logger.log(Level.WARNING, "Invalid key specified: " + runKeyString); - return; - } - String testName = testRunKey.getParent().getName(); + Key testRunKey; + try { + testRunKey = KeyFactory.stringToKey(runKeyString); + } catch (IllegalArgumentException e) { + logger.log(Level.WARNING, "Invalid key specified: " + runKeyString); + return; + } + String testName = testRunKey.getParent().getName(); - TestCoverageStatusEntity status = null; - Key statusKey = KeyFactory.createKey(TestCoverageStatusEntity.KIND, testName); - try { - status = TestCoverageStatusEntity.fromEntity(datastore.get(statusKey)); - } catch (EntityNotFoundException e) { - // no existing status - } - if (status == null) { - status = new TestCoverageStatusEntity(testName, 0, -1, -1); - } + TestCoverageStatusEntity status = ofy().load().type(TestCoverageStatusEntity.class).id(testName) + .now(); + if (status == null) { + status = new TestCoverageStatusEntity(testName, 0, -1, -1); + } - StringBuffer fullUrl = request.getRequestURL(); - String baseUrl = fullUrl.substring(0, fullUrl.indexOf(request.getRequestURI())); - String link = baseUrl + "/show_tree?testName=" + testName; - TestCoverageStatusEntity newStatus; - List<Message> messageQueue = new ArrayList<>(); - try { - List<String> emails = EmailHelper.getSubscriberEmails(testRunKey.getParent()); - newStatus = getTestCoverageStatus(status, testRunKey, link, emails, messageQueue); - } catch (IOException e) { - logger.log(Level.SEVERE, e.toString()); - return; - } + StringBuffer fullUrl = request.getRequestURL(); + String baseUrl = fullUrl.substring(0, fullUrl.indexOf(request.getRequestURI())); + String link = baseUrl + "/show_tree?testName=" + testName; + TestCoverageStatusEntity newStatus; + List<Message> messageQueue = new ArrayList<>(); + try { + List<String> emails = EmailHelper.getSubscriberEmails(testRunKey.getParent()); + newStatus = getTestCoverageStatus(status, testRunKey, link, emails, messageQueue); + } catch (IOException e) { + logger.log(Level.SEVERE, e.toString()); + return; + } - if (newStatus == null) { - return; - } - int retries = 0; - while (true) { - Transaction txn = datastore.beginTransaction(); - try { - try { - status = TestCoverageStatusEntity.fromEntity(datastore.get(statusKey)); - } catch (EntityNotFoundException e) { - // no status left - } - if (status == null || status.timestamp < newStatus.timestamp) { - datastore.put(newStatus.toEntity()); - txn.commit(); - EmailHelper.sendAll(messageQueue); - } else { - txn.rollback(); - } - break; - } catch (ConcurrentModificationException - | DatastoreFailureException - | DatastoreTimeoutException e) { - logger.log(Level.WARNING, "Retrying alert job insert: " + statusKey); - if (retries++ >= DatastoreHelper.MAX_WRITE_RETRIES) { - logger.log(Level.SEVERE, "Exceeded alert job retries: " + statusKey); - throw e; - } - } finally { - if (txn.isActive()) { - txn.rollback(); - } - } - } + if (newStatus == null) { + return; + } else { + if (status == null || status.getUpdatedTimestamp() < newStatus.getUpdatedTimestamp()) { + newStatus.save(); + EmailHelper.sendAll(messageQueue); + } } + } } diff --git a/src/main/java/com/android/vts/servlet/BaseServlet.java b/src/main/java/com/android/vts/servlet/BaseServlet.java index 045c051..96ba561 100644 --- a/src/main/java/com/android/vts/servlet/BaseServlet.java +++ b/src/main/java/com/android/vts/servlet/BaseServlet.java @@ -16,6 +16,7 @@ package com.android.vts.servlet; +import com.android.vts.entity.CoverageEntity; import com.android.vts.entity.TestSuiteResultEntity; import com.android.vts.entity.UserEntity; import com.android.vts.util.EmailHelper; @@ -158,6 +159,7 @@ public abstract class BaseServlet extends HttpServlet { CLIENT_ID = systemConfigProp.getProperty("appengine.clientID"); ANALYTICS_ID = systemConfigProp.getProperty("analytics.id"); + CoverageEntity.setPropertyValues(systemConfigProp); TestSuiteResultEntity.setPropertyValues(systemConfigProp); EmailHelper.setPropertyValues(systemConfigProp); GcsHelper.setGcsProjectId(systemConfigProp.getProperty("gcs.projectID")); diff --git a/src/main/java/com/android/vts/servlet/DashboardMainServlet.java b/src/main/java/com/android/vts/servlet/DashboardMainServlet.java index 5705c71..634f892 100644 --- a/src/main/java/com/android/vts/servlet/DashboardMainServlet.java +++ b/src/main/java/com/android/vts/servlet/DashboardMainServlet.java @@ -40,12 +40,15 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.logging.Level; +import java.util.stream.Collectors; import javax.servlet.RequestDispatcher; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; +import static com.googlecode.objectify.ObjectifyService.ofy; + /** Represents the servlet that is invoked on loading the first page of dashboard. */ public class DashboardMainServlet extends BaseServlet { private static final String DASHBOARD_MAIN_JSP = "WEB-INF/jsp/dashboard_main.jsp"; @@ -165,7 +168,6 @@ public class DashboardMainServlet extends BaseServlet { } List<TestDisplay> displayedTests = new ArrayList<>(); - List<String> allTestNames = new ArrayList<>(); List<Key> unprocessedTestKeys = new ArrayList<>(); Map<Key, TestDisplay> testMap = new HashMap<>(); // map from table key to TestDisplay @@ -174,10 +176,7 @@ public class DashboardMainServlet extends BaseServlet { boolean showAll = request.getParameter("showAll") != null; String error = null; - Query query = new Query(TestEntity.KIND).setKeysOnly(); - for (Entity test : datastore.prepare(query).asIterable()) { - allTestNames.add(test.getKey().getName()); - } + List<String> allTestNames = TestEntity.getAllTestNames(); List<Key> favoriteKeyList = new ArrayList<Key>(); Filter userFilter = @@ -191,7 +190,7 @@ public class DashboardMainServlet extends BaseServlet { subscriptionMap.put(testKey.getName(), KeyFactory.keyToString(fe.getKey())); }); - query = + Query query = new Query(TestStatusEntity.KIND) .addProjection( new PropertyProjection(TestStatusEntity.PASS_COUNT, Long.class)) diff --git a/src/main/java/com/android/vts/servlet/ShowCoverageOverviewServlet.java b/src/main/java/com/android/vts/servlet/ShowCoverageOverviewServlet.java index f9240c0..2d09c0b 100644 --- a/src/main/java/com/android/vts/servlet/ShowCoverageOverviewServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowCoverageOverviewServlet.java @@ -16,6 +16,7 @@ package com.android.vts.servlet; +import com.android.vts.entity.TestCoverageStatusEntity; import com.android.vts.entity.TestEntity; import com.android.vts.entity.TestRunEntity; import com.android.vts.proto.VtsReportMessage; @@ -34,128 +35,139 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.logging.Level; +import java.util.stream.Collectors; import javax.servlet.RequestDispatcher; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -/** Represents the servlet that is invoked on loading the coverage overview page. */ +/** + * Represents the servlet that is invoked on loading the coverage overview page. + */ public class ShowCoverageOverviewServlet extends BaseServlet { - private static final String COVERAGE_OVERVIEW_JSP = "WEB-INF/jsp/show_coverage_overview.jsp"; - @Override - public PageType getNavParentType() { - return PageType.COVERAGE_OVERVIEW; + private static final String COVERAGE_OVERVIEW_JSP = "WEB-INF/jsp/show_coverage_overview.jsp"; + + @Override + public PageType getNavParentType() { + return PageType.COVERAGE_OVERVIEW; + } + + @Override + public List<Page> getBreadcrumbLinks(HttpServletRequest request) { + return null; + } + + @Override + public void doGetHandler(HttpServletRequest request, HttpServletResponse response) + throws IOException { + RequestDispatcher dispatcher = null; + DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); + boolean unfiltered = request.getParameter("unfiltered") != null; + boolean showPresubmit = request.getParameter("showPresubmit") != null; + boolean showPostsubmit = request.getParameter("showPostsubmit") != null; + + // If no params are specified, set to default of postsubmit-only. + if (!(showPresubmit || showPostsubmit)) { + showPostsubmit = true; } - @Override - public List<Page> getBreadcrumbLinks(HttpServletRequest request) { - return null; + // If unfiltered, set showPre- and Post-submit to true for accurate UI. + if (unfiltered) { + showPostsubmit = true; + showPresubmit = true; } - @Override - public void doGetHandler(HttpServletRequest request, HttpServletResponse response) - throws IOException { - RequestDispatcher dispatcher = null; - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - boolean unfiltered = request.getParameter("unfiltered") != null; - boolean showPresubmit = request.getParameter("showPresubmit") != null; - boolean showPostsubmit = request.getParameter("showPostsubmit") != null; - - // If no params are specified, set to default of postsubmit-only. - if (!(showPresubmit || showPostsubmit)) { - showPostsubmit = true; - } - - // If unfiltered, set showPre- and Post-submit to true for accurate UI. - if (unfiltered) { - showPostsubmit = true; - showPresubmit = true; - } + Map<String, TestCoverageStatusEntity> testCoverageStatusMap = TestCoverageStatusEntity + .getTestCoverageStatusMap(); - Query q = new Query(TestEntity.KIND).setKeysOnly(); - List<Key> allTests = new ArrayList<>(); - for (Entity test : datastore.prepare(q).asIterable()) { - allTests.add(test.getKey()); - } + List<Key> allTests = TestEntity.getAllTest().stream().map(t -> t.getOldKey()) + .collect(Collectors.toList()); - // Add test names to list - List<String> resultNames = new ArrayList<>(); - for (VtsReportMessage.TestCaseResult r : VtsReportMessage.TestCaseResult.values()) { - resultNames.add(r.name()); - } + // Add test names to list + List<String> resultNames = new ArrayList<>(); + for (VtsReportMessage.TestCaseResult r : VtsReportMessage.TestCaseResult.values()) { + resultNames.add(r.name()); + } - List<JsonObject> testRunObjects = new ArrayList<>(); + List<JsonObject> testRunObjects = new ArrayList<>(); - Query.Filter testFilter = - new Query.FilterPredicate( - TestRunEntity.HAS_COVERAGE, Query.FilterOperator.EQUAL, true); - Query.Filter timeFilter = - FilterUtil.getTestTypeFilter(showPresubmit, showPostsubmit, unfiltered); + Query.Filter testFilter = + new Query.FilterPredicate( + TestRunEntity.HAS_COVERAGE, Query.FilterOperator.EQUAL, true); + Query.Filter timeFilter = + FilterUtil.getTestTypeFilter(showPresubmit, showPostsubmit, unfiltered); - if (timeFilter != null) { - testFilter = Query.CompositeFilterOperator.and(testFilter, timeFilter); + if (timeFilter != null) { + testFilter = Query.CompositeFilterOperator.and(testFilter, timeFilter); + } + Map<String, String[]> parameterMap = request.getParameterMap(); + List<Query.Filter> userTestFilters = FilterUtil.getUserTestFilters(parameterMap); + userTestFilters.add(0, testFilter); + Query.Filter userDeviceFilter = FilterUtil.getUserDeviceFilter(parameterMap); + + int coveredLines = 0; + int uncoveredLines = 0; + int passCount = 0; + int failCount = 0; + for (Key key : allTests) { + List<Key> gets = + FilterUtil.getMatchingKeys( + key, + TestRunEntity.KIND, + userTestFilters, + userDeviceFilter, + Query.SortDirection.DESCENDING, + 1); + Map<Key, Entity> entityMap = datastore.get(gets); + for (Key entityKey : gets) { + if (!entityMap.containsKey(entityKey)) { + continue; } - Map<String, String[]> parameterMap = request.getParameterMap(); - List<Query.Filter> userTestFilters = FilterUtil.getUserTestFilters(parameterMap); - userTestFilters.add(0, testFilter); - Query.Filter userDeviceFilter = FilterUtil.getUserDeviceFilter(parameterMap); - - int coveredLines = 0; - int uncoveredLines = 0; - int passCount = 0; - int failCount = 0; - for (Key key : allTests) { - List<Key> gets = - FilterUtil.getMatchingKeys( - key, - TestRunEntity.KIND, - userTestFilters, - userDeviceFilter, - Query.SortDirection.DESCENDING, - 1); - Map<Key, Entity> entityMap = datastore.get(gets); - for (Key entityKey : gets) { - if (!entityMap.containsKey(entityKey)) { - continue; - } - TestRunEntity testRunEntity = TestRunEntity.fromEntity(entityMap.get(entityKey)); - if (testRunEntity == null) { - continue; - } - TestRunMetadata metadata = new TestRunMetadata(key.getName(), testRunEntity); - testRunObjects.add(metadata.toJson()); - coveredLines += testRunEntity.coveredLineCount; - uncoveredLines += testRunEntity.totalLineCount - testRunEntity.coveredLineCount; - passCount += testRunEntity.passCount; - failCount += testRunEntity.failCount; - } + TestRunEntity testRunEntity = TestRunEntity.fromEntity(entityMap.get(entityKey)); + if (testRunEntity == null) { + continue; } - FilterUtil.setAttributes(request, parameterMap); - - int[] testStats = new int[VtsReportMessage.TestCaseResult.values().length]; - testStats[VtsReportMessage.TestCaseResult.TEST_CASE_RESULT_PASS.getNumber()] = passCount; - testStats[VtsReportMessage.TestCaseResult.TEST_CASE_RESULT_FAIL.getNumber()] = failCount; - - response.setStatus(HttpServletResponse.SC_OK); - request.setAttribute("resultNames", resultNames); - request.setAttribute("resultNamesJson", new Gson().toJson(resultNames)); - request.setAttribute("testRuns", new Gson().toJson(testRunObjects)); - request.setAttribute("coveredLines", new Gson().toJson(coveredLines)); - request.setAttribute("uncoveredLines", new Gson().toJson(uncoveredLines)); - request.setAttribute("testStats", new Gson().toJson(testStats)); - - request.setAttribute("unfiltered", unfiltered); - request.setAttribute("showPresubmit", showPresubmit); - request.setAttribute("showPostsubmit", showPostsubmit); - request.setAttribute("branches", new Gson().toJson(DatastoreHelper.getAllBranches())); - request.setAttribute("devices", new Gson().toJson(DatastoreHelper.getAllBuildFlavors())); - dispatcher = request.getRequestDispatcher(COVERAGE_OVERVIEW_JSP); - try { - dispatcher.forward(request, response); - } catch (ServletException e) { - logger.log(Level.SEVERE, "Servlet Exception caught : ", e); - } + // Overwrite the coverage value with newly update value from user decision + TestCoverageStatusEntity testCoverageStatusEntity = testCoverageStatusMap + .get(key.getName()); + testRunEntity.setCoveredLineCount(testCoverageStatusEntity.getUpdatedCoveredLineCount()); + testRunEntity.setTotalLineCount(testCoverageStatusEntity.getUpdatedTotalLineCount()); + TestRunMetadata metadata = new TestRunMetadata(key.getName(), testRunEntity); + + testRunObjects.add(metadata.toJson()); + coveredLines += testRunEntity.getCoveredLineCount(); + uncoveredLines += testRunEntity.getTotalLineCount() - testRunEntity.getCoveredLineCount(); + passCount += testRunEntity.getPassCount(); + failCount += testRunEntity.getFailCount(); + } + } + + FilterUtil.setAttributes(request, parameterMap); + + int[] testStats = new int[VtsReportMessage.TestCaseResult.values().length]; + testStats[VtsReportMessage.TestCaseResult.TEST_CASE_RESULT_PASS.getNumber()] = passCount; + testStats[VtsReportMessage.TestCaseResult.TEST_CASE_RESULT_FAIL.getNumber()] = failCount; + + response.setStatus(HttpServletResponse.SC_OK); + request.setAttribute("resultNames", resultNames); + request.setAttribute("resultNamesJson", new Gson().toJson(resultNames)); + request.setAttribute("testRuns", new Gson().toJson(testRunObjects)); + request.setAttribute("coveredLines", new Gson().toJson(coveredLines)); + request.setAttribute("uncoveredLines", new Gson().toJson(uncoveredLines)); + request.setAttribute("testStats", new Gson().toJson(testStats)); + + request.setAttribute("unfiltered", unfiltered); + request.setAttribute("showPresubmit", showPresubmit); + request.setAttribute("showPostsubmit", showPostsubmit); + request.setAttribute("branches", new Gson().toJson(DatastoreHelper.getAllBranches())); + request.setAttribute("devices", new Gson().toJson(DatastoreHelper.getAllBuildFlavors())); + dispatcher = request.getRequestDispatcher(COVERAGE_OVERVIEW_JSP); + try { + dispatcher.forward(request, response); + } catch (ServletException e) { + logger.log(Level.SEVERE, "Servlet Exception caught : ", e); } + } } diff --git a/src/main/java/com/android/vts/servlet/ShowCoverageServlet.java b/src/main/java/com/android/vts/servlet/ShowCoverageServlet.java index 0b889d5..08e5db6 100644 --- a/src/main/java/com/android/vts/servlet/ShowCoverageServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowCoverageServlet.java @@ -16,9 +16,13 @@ package com.android.vts.servlet; +import static com.googlecode.objectify.ObjectifyService.ofy; + import com.android.vts.entity.CoverageEntity; +import com.android.vts.entity.RoleEntity; import com.android.vts.entity.TestEntity; import com.android.vts.entity.TestRunEntity; +import com.android.vts.entity.UserEntity; import com.google.appengine.api.datastore.DatastoreService; import com.google.appengine.api.datastore.DatastoreServiceFactory; import com.google.appengine.api.datastore.Entity; @@ -26,117 +30,94 @@ import com.google.appengine.api.datastore.Key; import com.google.appengine.api.datastore.KeyFactory; import com.google.appengine.api.datastore.Query; import com.google.gson.Gson; +import com.googlecode.objectify.Ref; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.logging.Level; import javax.servlet.RequestDispatcher; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -/** Servlet for handling requests to show code coverage. */ +/** + * Servlet for handling requests to show code coverage. + */ public class ShowCoverageServlet extends BaseServlet { - private static final String COVERAGE_JSP = "WEB-INF/jsp/show_coverage.jsp"; - private static final String TREE_JSP = "WEB-INF/jsp/show_tree.jsp"; - @Override - public PageType getNavParentType() { - return PageType.TOT; - } + private static final String COVERAGE_JSP = "WEB-INF/jsp/show_coverage.jsp"; + private static final String TREE_JSP = "WEB-INF/jsp/show_tree.jsp"; + + @Override + public PageType getNavParentType() { + return PageType.TOT; + } + + @Override + public List<Page> getBreadcrumbLinks(HttpServletRequest request) { + List<Page> links = new ArrayList<>(); + String testName = request.getParameter("testName"); + links.add(new Page(PageType.TABLE, testName, "?testName=" + testName)); + + String startTime = request.getParameter("startTime"); + links.add(new Page(PageType.COVERAGE, "?testName=" + testName + "&startTime=" + startTime)); + return links; + } + + @Override + public void doGetHandler(HttpServletRequest request, HttpServletResponse response) + throws IOException { + RequestDispatcher dispatcher = null; - @Override - public List<Page> getBreadcrumbLinks(HttpServletRequest request) { - List<Page> links = new ArrayList<>(); - String testName = request.getParameter("testName"); - links.add(new Page(PageType.TABLE, testName, "?testName=" + testName)); + String test = request.getParameter("testName"); + String timeString = request.getParameter("startTime"); - String startTime = request.getParameter("startTime"); - links.add(new Page(PageType.COVERAGE, "?testName=" + testName + "&startTime=" + startTime)); - return links; + Boolean isModerator = false; + String currentUserEmail = request.getAttribute("email").toString(); + Optional<UserEntity> userEntityOptional = Optional + .ofNullable(UserEntity.getUser(currentUserEmail)); + if (userEntityOptional.isPresent()) { + Ref refRole = Ref.create(RoleEntity.getRole("coverage-moderator")); + isModerator = userEntityOptional.get().getRoles().contains(refRole); } - @Override - public void doGetHandler(HttpServletRequest request, HttpServletResponse response) - throws IOException { - RequestDispatcher dispatcher = null; - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - String test = request.getParameter("testName"); - String timeString = request.getParameter("startTime"); - - // Process the time key requested - long time = -1; - try { - time = Long.parseLong(timeString); - } catch (NumberFormatException e) { - request.setAttribute("testName", test); - dispatcher = request.getRequestDispatcher(TREE_JSP); - return; - } - - // Compute the parent test run key based off of the test and time - Key testKey = KeyFactory.createKey(TestEntity.KIND, test); - Key testRunKey = KeyFactory.createKey(testKey, TestRunEntity.KIND, time); - - // Create a query for coverage entities - Query coverageQuery = new Query(CoverageEntity.KIND).setAncestor(testRunKey); - - List<String> sourceFiles = new ArrayList<>(); // list of source files - List<List<Long>> coverageVectors = new ArrayList<>(); // list of line coverage vectors - List<String> projects = new ArrayList<>(); // list of project names - List<String> commits = new ArrayList<>(); // list of project commit hashes - List<String> indicators = new ArrayList<>(); // list of HTML indicates to display - - /* - * Map from section name to a list of indexes into the above lists where each coverage - * report's data is located. - */ - Map<String, List<Integer>> sectionMap = new HashMap<>(); - for (Entity e : datastore.prepare(coverageQuery).asIterable()) { - CoverageEntity coverageEntity = CoverageEntity.fromEntity(e); - if (coverageEntity == null) { - logger.log(Level.WARNING, "Invalid coverage entity: " + e.getKey()); - continue; - } - if (!sectionMap.containsKey(coverageEntity.group)) { - sectionMap.put(coverageEntity.group, new ArrayList<Integer>()); - } - sectionMap.get(coverageEntity.group).add(coverageVectors.size()); - coverageVectors.add(coverageEntity.lineCoverage); - sourceFiles.add(coverageEntity.filePath); - projects.add(coverageEntity.projectName); - commits.add(coverageEntity.projectVersion); - String indicator = ""; - long total = coverageEntity.totalLineCount; - long covered = coverageEntity.coveredLineCount; - if (total > 0) { - double pct = Math.round(covered * 10000d / total) / 100d; - String color = pct >= 70 ? "green" : "red"; - indicator = "<div class=\"right total-count\">" + covered + "/" + total + "</div>" - + "<div class=\"indicator " + color + "\">" + pct + "%</div>"; - } - indicators.add(indicator); - } - - request.setAttribute("testName", request.getParameter("testName")); - request.setAttribute("gerritURI", new Gson().toJson(GERRIT_URI)); - request.setAttribute("gerritScope", new Gson().toJson(GERRIT_SCOPE)); - request.setAttribute("clientId", new Gson().toJson(CLIENT_ID)); - request.setAttribute("coverageVectors", new Gson().toJson(coverageVectors)); - request.setAttribute("sourceFiles", new Gson().toJson(sourceFiles)); - request.setAttribute("projects", new Gson().toJson(projects)); - request.setAttribute("commits", new Gson().toJson(commits)); - request.setAttribute("indicators", new Gson().toJson(indicators)); - request.setAttribute("sectionMap", new Gson().toJson(sectionMap)); - request.setAttribute("startTime", request.getParameter("startTime")); - dispatcher = request.getRequestDispatcher(COVERAGE_JSP); - - try { - dispatcher.forward(request, response); - } catch (ServletException e) { - logger.log(Level.SEVERE, "Servlet Exception caught : ", e); - } + // Process the time key requested + long time = -1; + try { + time = Long.parseLong(timeString); + } catch (NumberFormatException e) { + request.setAttribute("testName", test); + dispatcher = request.getRequestDispatcher(TREE_JSP); + return; + } + + com.googlecode.objectify.Key testKey = com.googlecode.objectify.Key + .create(TestEntity.class, test); + com.googlecode.objectify.Key testRunKey = com.googlecode.objectify.Key + .create(testKey, TestRunEntity.class, time); + + List<CoverageEntity> coverageEntityList = ofy().load().type(CoverageEntity.class) + .ancestor(testRunKey).list(); + + Collections.sort(coverageEntityList, CoverageEntity.isIgnoredComparator); + + request.setAttribute("isModerator", isModerator); + request.setAttribute("testName", request.getParameter("testName")); + request.setAttribute("gerritURI", new Gson().toJson(GERRIT_URI)); + request.setAttribute("gerritScope", new Gson().toJson(GERRIT_SCOPE)); + request.setAttribute("clientId", new Gson().toJson(CLIENT_ID)); + request.setAttribute("startTime", request.getParameter("startTime")); + request.setAttribute("coverageEntityList", coverageEntityList); + + dispatcher = request.getRequestDispatcher(COVERAGE_JSP); + try { + dispatcher.forward(request, response); + } catch (ServletException e) { + logger.log(Level.SEVERE, "Servlet Exception caught : ", e); } + } } diff --git a/src/main/java/com/android/vts/servlet/ShowGreenReleaseServlet.java b/src/main/java/com/android/vts/servlet/ShowGreenReleaseServlet.java index 43ebe3a..7fad9a2 100644 --- a/src/main/java/com/android/vts/servlet/ShowGreenReleaseServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowGreenReleaseServlet.java @@ -325,8 +325,8 @@ public class ShowGreenReleaseServlet extends BaseServlet { List<String> passBuildIdList = testPlanRunEntityList .stream() - .filter(entity -> entity.failCount == 0L) - .map(entity -> entity.testBuildId) + .filter(entity -> entity.getFailCount() == 0L) + .map(entity -> entity.getTestBuildId()) .collect(Collectors.toList()); allPassIdLists.add(passBuildIdList); logger.log(Level.INFO, "passBuildIdList => " + passBuildIdList); @@ -334,11 +334,11 @@ public class ShowGreenReleaseServlet extends BaseServlet { // The logic for candidate build ID is starting from here Comparator<TestPlanRunEntity> byPassing = Comparator.comparingLong( - elemFirst -> elemFirst.passCount); + elemFirst -> elemFirst.getPassCount()); Comparator<TestPlanRunEntity> byNonPassing = Comparator.comparingLong( - elemFirst -> elemFirst.failCount); + elemFirst -> elemFirst.getFailCount()); // This will get the TestPlanRunEntity having maximum number of // passing and minimum number of fail @@ -353,14 +353,14 @@ public class ShowGreenReleaseServlet extends BaseServlet { String buildId = testPlanRunEntity - .map(entity -> entity.testBuildId) + .map(entity -> entity.getTestBuildId()) .orElse(""); deviceBuildInfo.setCandidateBuildId(buildId); Long buildIdTimestamp = testPlanRunEntity .map( entity -> { - return entity.startTimestamp; + return entity.getStartTimestamp(); }) .orElse(0L); deviceBuildInfo.setCandidateBuildIdTimestamp(buildIdTimestamp); @@ -385,8 +385,8 @@ public class ShowGreenReleaseServlet extends BaseServlet { .stream() .filter( entity -> - entity.failCount == 0L - && entity.testBuildId + entity.getFailCount() == 0L + && entity.getTestBuildId() .equalsIgnoreCase( greenBuildId)) .findFirst(); @@ -395,7 +395,7 @@ public class ShowGreenReleaseServlet extends BaseServlet { deviceBuildInfo.setGreenBuildId(greenBuildId); Long buildIdTimestamp = testPlanRunEntity - .map(entity -> entity.startTimestamp) + .map(entity -> entity.getStartTimestamp()) .orElse(0L); deviceBuildInfo.setGreenBuildIdTimestamp(buildIdTimestamp); }); diff --git a/src/main/java/com/android/vts/servlet/ShowPlanReleaseServlet.java b/src/main/java/com/android/vts/servlet/ShowPlanReleaseServlet.java index 96012aa..fda4c13 100644 --- a/src/main/java/com/android/vts/servlet/ShowPlanReleaseServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowPlanReleaseServlet.java @@ -102,8 +102,8 @@ public class ShowPlanReleaseServlet extends BaseServlet { @Override public int compareTo(TestPlanRunMetadata o) { - return new Long(o.testPlanRun.startTimestamp) - .compareTo(this.testPlanRun.startTimestamp); + return new Long(o.testPlanRun.getStartTimestamp()) + .compareTo(this.testPlanRun.getStartTimestamp()); } } @@ -246,10 +246,10 @@ public class ShowPlanReleaseServlet extends BaseServlet { if (testPlanRuns.size() > 0) { TestPlanRunMetadata firstRun = testPlanRuns.get(0); - endTime = firstRun.testPlanRun.startTimestamp; + endTime = firstRun.testPlanRun.getStartTimestamp(); TestPlanRunMetadata lastRun = testPlanRuns.get(testPlanRuns.size() - 1); - startTime = lastRun.testPlanRun.startTimestamp; + startTime = lastRun.testPlanRun.getStartTimestamp(); } List<JsonObject> testPlanRunObjects = new ArrayList<>(); diff --git a/src/main/java/com/android/vts/servlet/ShowPlanRunServlet.java b/src/main/java/com/android/vts/servlet/ShowPlanRunServlet.java index 442d428..bce89fc 100644 --- a/src/main/java/com/android/vts/servlet/ShowPlanRunServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowPlanRunServlet.java @@ -101,15 +101,15 @@ public class ShowPlanRunServlet extends BaseServlet { try { Entity testPlanRunEntity = datastore.get(planRunKey); TestPlanRunEntity testPlanRun = TestPlanRunEntity.fromEntity(testPlanRunEntity); - Map<Key, Entity> testRuns = datastore.get(testPlanRun.testRuns); - testBuildId = testPlanRun.testBuildId; - passCount = (int) testPlanRun.passCount; - failCount = (int) testPlanRun.failCount; - startTime = testPlanRun.startTimestamp; - endTime = testPlanRun.endTimestamp; - moduleCount = testPlanRun.testRuns.size(); - - for (Key key : testPlanRun.testRuns) { + Map<Key, Entity> testRuns = datastore.get(testPlanRun.getTestRuns()); + testBuildId = testPlanRun.getTestBuildId(); + passCount = (int) testPlanRun.getPassCount(); + failCount = (int) testPlanRun.getFailCount(); + startTime = testPlanRun.getStartTimestamp(); + endTime = testPlanRun.getEndTimestamp(); + moduleCount = testPlanRun.getTestRuns().size(); + + for (Key key : testPlanRun.getTestRuns()) { if (!testRuns.containsKey(key)) continue; TestRunEntity testRunEntity = TestRunEntity.fromEntity(testRuns.get(key)); if (testRunEntity == null) continue; @@ -122,7 +122,7 @@ public class ShowPlanRunServlet extends BaseServlet { } TestRunMetadata metadata = new TestRunMetadata(key.getParent().getName(), testRunEntity, devices); - if (metadata.testRun.failCount > 0) { + if (metadata.testRun.getFailCount() > 0) { failingTestObjects.add(metadata.toJson()); } else { passingTestObjects.add(metadata.toJson()); diff --git a/src/main/java/com/android/vts/servlet/ShowProfilingListServlet.java b/src/main/java/com/android/vts/servlet/ShowProfilingListServlet.java index cbebd2c..ce138cc 100644 --- a/src/main/java/com/android/vts/servlet/ShowProfilingListServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowProfilingListServlet.java @@ -16,6 +16,8 @@ package com.android.vts.servlet; +import static com.googlecode.objectify.ObjectifyService.ofy; + import com.android.vts.entity.TestEntity; import com.google.appengine.api.datastore.DatastoreService; import com.google.appengine.api.datastore.DatastoreServiceFactory; @@ -28,49 +30,44 @@ import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.logging.Level; +import java.util.stream.Collectors; import javax.servlet.RequestDispatcher; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -/** Servlet for handling requests to display profiling tests. */ +/** + * Servlet for handling requests to display profiling tests. + */ public class ShowProfilingListServlet extends BaseServlet { - private static final String PROFILING_LIST_JSP = "WEB-INF/jsp/show_profiling_list.jsp"; - @Override - public PageType getNavParentType() { - return PageType.PROFILING_LIST; - } + private static final String PROFILING_LIST_JSP = "WEB-INF/jsp/show_profiling_list.jsp"; - @Override - public List<Page> getBreadcrumbLinks(HttpServletRequest request) { - return null; - } + @Override + public PageType getNavParentType() { + return PageType.PROFILING_LIST; + } - @Override - public void doGetHandler(HttpServletRequest request, HttpServletResponse response) - throws IOException { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - Query.Filter profilingFilter = new Query.FilterPredicate( - TestEntity.HAS_PROFILING_DATA, Query.FilterOperator.EQUAL, true); - Query query = new Query(TestEntity.KIND) - .setFilter(profilingFilter) - .setKeysOnly(); - Set<String> profilingTests = new HashSet<>(); - for (Entity test : datastore.prepare(query).asIterable()) { - profilingTests.add(test.getKey().getName()); - } + @Override + public List<Page> getBreadcrumbLinks(HttpServletRequest request) { + return null; + } - List<String> tests = new ArrayList<>(profilingTests); - tests.sort(Comparator.naturalOrder()); + @Override + public void doGetHandler(HttpServletRequest request, HttpServletResponse response) + throws IOException { + List<String> tests = ofy().load().type(TestEntity.class) + .filter(TestEntity.HAS_PROFILING_DATA, true).list().stream() + .sorted(Comparator.comparing(TestEntity::getTestName)).map(t -> t.getTestName()) + .collect(Collectors.toList()); - response.setStatus(HttpServletResponse.SC_OK); - request.setAttribute("testNames", tests); - RequestDispatcher dispatcher = request.getRequestDispatcher(PROFILING_LIST_JSP); - try { - dispatcher.forward(request, response); - } catch (ServletException e) { - logger.log(Level.SEVERE, "Servlet Excpetion caught : ", e); - } + response.setStatus(HttpServletResponse.SC_OK); + request.setAttribute("testNames", tests); + RequestDispatcher dispatcher = request.getRequestDispatcher(PROFILING_LIST_JSP); + try { + dispatcher.forward(request, response); + } catch (ServletException e) { + logger.log(Level.SEVERE, "Servlet Excpetion caught : ", e); } + } } diff --git a/src/main/java/com/android/vts/servlet/ShowReleaseServlet.java b/src/main/java/com/android/vts/servlet/ShowReleaseServlet.java index ed634a3..ae175ae 100644 --- a/src/main/java/com/android/vts/servlet/ShowReleaseServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowReleaseServlet.java @@ -39,76 +39,73 @@ import java.util.stream.Collectors; import static com.googlecode.objectify.ObjectifyService.ofy; -/** Represents the servlet that is invoked on loading the release page. */ +/** + * Represents the servlet that is invoked on loading the release page. + */ public class ShowReleaseServlet extends BaseServlet { - @Override - public PageType getNavParentType() { - return PageType.RELEASE; - } - - @Override - public List<Page> getBreadcrumbLinks(HttpServletRequest request) { - return null; - } - - @Override - public void doGetHandler(HttpServletRequest request, HttpServletResponse response) - throws IOException { - String testType = - request.getParameter("type") == null ? "plan" : request.getParameter("type"); - - RequestDispatcher dispatcher; - if (testType.equalsIgnoreCase("plan")) { - dispatcher = this.getTestPlanDispatcher(request, response); - } else { - dispatcher = this.getTestSuiteDispatcher(request, response); - } - - try { - request.setAttribute("testType", testType); - response.setStatus(HttpServletResponse.SC_OK); - dispatcher.forward(request, response); - } catch (ServletException e) { - logger.log(Level.SEVERE, "Servlet Excpetion caught : ", e); - } - } - - private RequestDispatcher getTestPlanDispatcher( - HttpServletRequest request, HttpServletResponse response) { - String RELEASE_JSP = "WEB-INF/jsp/show_release.jsp"; - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - - Set<String> planSet = new HashSet<>(); - - Query q = new Query(TestPlanEntity.KIND).setKeysOnly(); - for (Entity testPlanEntity : datastore.prepare(q).asIterable()) { - planSet.add(testPlanEntity.getKey().getName()); - } - - List<String> plans = new ArrayList<>(planSet); - plans.sort(Comparator.naturalOrder()); - - request.setAttribute("isAdmin", UserServiceFactory.getUserService().isUserAdmin()); - request.setAttribute("planNames", plans); - RequestDispatcher dispatcher = request.getRequestDispatcher(RELEASE_JSP); - return dispatcher; + @Override + public PageType getNavParentType() { + return PageType.RELEASE; + } + + @Override + public List<Page> getBreadcrumbLinks(HttpServletRequest request) { + return null; + } + + @Override + public void doGetHandler(HttpServletRequest request, HttpServletResponse response) + throws IOException { + String testType = + request.getParameter("type") == null ? "plan" : request.getParameter("type"); + + RequestDispatcher dispatcher; + if (testType.equalsIgnoreCase("plan")) { + dispatcher = this.getTestPlanDispatcher(request, response); + } else { + dispatcher = this.getTestSuiteDispatcher(request, response); } - private RequestDispatcher getTestSuiteDispatcher( - HttpServletRequest request, HttpServletResponse response) { - String RELEASE_JSP = "WEB-INF/jsp/show_release.jsp"; - - List<TestSuiteResultEntity> suiteResultEntityList = TestSuiteResultEntity.getTestSuitePlans(); - - List<String> plans = - suiteResultEntityList - .stream() - .map(suiteEntity -> suiteEntity.getSuitePlan()) - .collect(Collectors.toList()); - request.setAttribute("isAdmin", UserServiceFactory.getUserService().isUserAdmin()); - request.setAttribute("planNames", plans); - RequestDispatcher dispatcher = request.getRequestDispatcher(RELEASE_JSP); - return dispatcher; + try { + request.setAttribute("testType", testType); + response.setStatus(HttpServletResponse.SC_OK); + dispatcher.forward(request, response); + } catch (ServletException e) { + logger.log(Level.SEVERE, "Servlet Excpetion caught : ", e); } + } + + private RequestDispatcher getTestPlanDispatcher( + HttpServletRequest request, HttpServletResponse response) { + String RELEASE_JSP = "WEB-INF/jsp/show_release.jsp"; + + List<TestPlanEntity> testPlanEntityList = ofy().load().type(TestPlanEntity.class).list(); + + List<String> plans = testPlanEntityList.stream() + .sorted(Comparator.comparing(TestPlanEntity::getTestPlanName)) + .map(te -> te.getTestPlanName()).collect(Collectors.toList()); + + request.setAttribute("isAdmin", UserServiceFactory.getUserService().isUserAdmin()); + request.setAttribute("planNames", plans); + RequestDispatcher dispatcher = request.getRequestDispatcher(RELEASE_JSP); + return dispatcher; + } + + private RequestDispatcher getTestSuiteDispatcher( + HttpServletRequest request, HttpServletResponse response) { + String RELEASE_JSP = "WEB-INF/jsp/show_release.jsp"; + + List<TestSuiteResultEntity> suiteResultEntityList = TestSuiteResultEntity.getTestSuitePlans(); + + List<String> plans = + suiteResultEntityList + .stream() + .map(suiteEntity -> suiteEntity.getSuitePlan()) + .collect(Collectors.toList()); + request.setAttribute("isAdmin", UserServiceFactory.getUserService().isUserAdmin()); + request.setAttribute("planNames", plans); + RequestDispatcher dispatcher = request.getRequestDispatcher(RELEASE_JSP); + return dispatcher; + } } diff --git a/src/main/java/com/android/vts/servlet/ShowTableServlet.java b/src/main/java/com/android/vts/servlet/ShowTableServlet.java index 0afaadf..cfe85c3 100644 --- a/src/main/java/com/android/vts/servlet/ShowTableServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowTableServlet.java @@ -70,7 +70,7 @@ public class ShowTableServlet extends BaseServlet { DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); List<Key> gets = new ArrayList<>(); - for (long testCaseId : testRunEntity.testCaseIds) { + for (long testCaseId : testRunEntity.getTestCaseIds()) { gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); } diff --git a/src/main/java/com/android/vts/servlet/ShowTestAcknowledgmentServlet.java b/src/main/java/com/android/vts/servlet/ShowTestAcknowledgmentServlet.java index 5703b94..c75352c 100644 --- a/src/main/java/com/android/vts/servlet/ShowTestAcknowledgmentServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowTestAcknowledgmentServlet.java @@ -62,11 +62,7 @@ public class ShowTestAcknowledgmentServlet extends BaseServlet { testAcks.add(ack.toJson()); } - List<String> allTestNames = new ArrayList<>(); - Query query = new Query(TestEntity.KIND).setKeysOnly(); - for (Entity test : datastore.prepare(query).asIterable()) { - allTestNames.add(test.getKey().getName()); - } + List<String> allTestNames = TestEntity.getAllTestNames(); request.setAttribute("testAcknowledgments", new Gson().toJson(testAcks)); request.setAttribute("allTests", new Gson().toJson(allTestNames)); diff --git a/src/main/java/com/android/vts/servlet/ShowTreeServlet.java b/src/main/java/com/android/vts/servlet/ShowTreeServlet.java index 0338bf9..d0a682e 100644 --- a/src/main/java/com/android/vts/servlet/ShowTreeServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowTreeServlet.java @@ -50,273 +50,280 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -/** Servlet for handling requests to load individual tables. */ +/** + * Servlet for handling requests to load individual tables. + */ public class ShowTreeServlet extends BaseServlet { - private static final String TABLE_JSP = "WEB-INF/jsp/show_tree.jsp"; - // Error message displayed on the webpage is tableName passed is null. - private static final String TABLE_NAME_ERROR = "Error : Table name must be passed!"; - private static final String PROFILING_DATA_ALERT = "No profiling data was found."; - private static final int MAX_RESULT_COUNT = 60; - private static final int MAX_PREFETCH_COUNT = 10; - - @Override - public PageType getNavParentType() { - return PageType.TOT; - } - @Override - public List<Page> getBreadcrumbLinks(HttpServletRequest request) { - List<Page> links = new ArrayList<>(); - String testName = request.getParameter("testName"); - links.add(new Page(PageType.TREE, testName, "?testName=" + testName)); - return links; + private static final String TABLE_JSP = "WEB-INF/jsp/show_tree.jsp"; + // Error message displayed on the webpage is tableName passed is null. + private static final String TABLE_NAME_ERROR = "Error : Table name must be passed!"; + private static final String PROFILING_DATA_ALERT = "No profiling data was found."; + private static final int MAX_RESULT_COUNT = 60; + private static final int MAX_PREFETCH_COUNT = 10; + + @Override + public PageType getNavParentType() { + return PageType.TOT; + } + + @Override + public List<Page> getBreadcrumbLinks(HttpServletRequest request) { + List<Page> links = new ArrayList<>(); + String testName = request.getParameter("testName"); + links.add(new Page(PageType.TREE, testName, "?testName=" + testName)); + return links; + } + + /** + * Get the test run details for a test run. + * + * @param metadata The metadata for the test run whose details will be fetched. + * @return The TestRunDetails object for the provided test run. + */ + public static TestRunDetails processTestDetails(TestRunMetadata metadata) { + DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); + TestRunDetails details = new TestRunDetails(); + List<Key> gets = new ArrayList<>(); + for (long testCaseId : metadata.testRun.getTestCaseIds()) { + gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); } - - /** - * Get the test run details for a test run. - * - * @param metadata The metadata for the test run whose details will be fetched. - * @return The TestRunDetails object for the provided test run. - */ - public static TestRunDetails processTestDetails(TestRunMetadata metadata) { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - TestRunDetails details = new TestRunDetails(); - List<Key> gets = new ArrayList<>(); - for (long testCaseId : metadata.testRun.testCaseIds) { - gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); - } - Map<Key, Entity> entityMap = datastore.get(gets); - for (int i = 0; i < 1; i++) { - for (Key key : entityMap.keySet()) { - TestCaseRunEntity testCaseRun = TestCaseRunEntity.fromEntity(entityMap.get(key)); - if (testCaseRun == null) { - continue; - } - details.addTestCase(testCaseRun); - } + Map<Key, Entity> entityMap = datastore.get(gets); + for (int i = 0; i < 1; i++) { + for (Key key : entityMap.keySet()) { + TestCaseRunEntity testCaseRun = TestCaseRunEntity.fromEntity(entityMap.get(key)); + if (testCaseRun == null) { + continue; } - return details; + details.addTestCase(testCaseRun); + } + } + return details; + } + + @Override + public void doGetHandler(HttpServletRequest request, HttpServletResponse response) + throws IOException { + boolean unfiltered = request.getParameter("unfiltered") != null; + boolean showPresubmit = request.getParameter("showPresubmit") != null; + boolean showPostsubmit = request.getParameter("showPostsubmit") != null; + Long startTime = null; // time in microseconds + Long endTime = null; // time in microseconds + DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); + RequestDispatcher dispatcher = null; + + // message to display if profiling point data is not available + String profilingDataAlert = ""; + + if (request.getParameter("testName") == null) { + request.setAttribute("testName", TABLE_NAME_ERROR); + return; + } + String testName = request.getParameter("testName"); + + if (request.getParameter("startTime") != null) { + String time = request.getParameter("startTime"); + try { + startTime = Long.parseLong(time); + startTime = startTime > 0 ? startTime : null; + } catch (NumberFormatException e) { + startTime = null; + } + } + if (request.getParameter("endTime") != null) { + String time = request.getParameter("endTime"); + try { + endTime = Long.parseLong(time); + endTime = endTime > 0 ? endTime : null; + } catch (NumberFormatException e) { + endTime = null; + } } - @Override - public void doGetHandler(HttpServletRequest request, HttpServletResponse response) - throws IOException { - boolean unfiltered = request.getParameter("unfiltered") != null; - boolean showPresubmit = request.getParameter("showPresubmit") != null; - boolean showPostsubmit = request.getParameter("showPostsubmit") != null; - Long startTime = null; // time in microseconds - Long endTime = null; // time in microseconds - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - RequestDispatcher dispatcher = null; - - // message to display if profiling point data is not available - String profilingDataAlert = ""; - - if (request.getParameter("testName") == null) { - request.setAttribute("testName", TABLE_NAME_ERROR); - return; - } - String testName = request.getParameter("testName"); - - if (request.getParameter("startTime") != null) { - String time = request.getParameter("startTime"); - try { - startTime = Long.parseLong(time); - startTime = startTime > 0 ? startTime : null; - } catch (NumberFormatException e) { - startTime = null; - } - } - if (request.getParameter("endTime") != null) { - String time = request.getParameter("endTime"); - try { - endTime = Long.parseLong(time); - endTime = endTime > 0 ? endTime : null; - } catch (NumberFormatException e) { - endTime = null; - } - } + // If no params are specified, set to default of postsubmit-only. + if (!(showPresubmit || showPostsubmit)) { + showPostsubmit = true; + } - // If no params are specified, set to default of postsubmit-only. - if (!(showPresubmit || showPostsubmit)) { - showPostsubmit = true; - } + // If unfiltered, set showPre- and Post-submit to true for accurate UI. + if (unfiltered) { + showPostsubmit = true; + showPresubmit = true; + } - // If unfiltered, set showPre- and Post-submit to true for accurate UI. - if (unfiltered) { - showPostsubmit = true; - showPresubmit = true; - } + // Add result names to list + List<String> resultNames = new ArrayList<>(); + for (TestCaseResult r : TestCaseResult.values()) { + resultNames.add(r.name()); + } - // Add result names to list - List<String> resultNames = new ArrayList<>(); - for (TestCaseResult r : TestCaseResult.values()) { - resultNames.add(r.name()); - } + SortDirection dir = SortDirection.DESCENDING; + if (startTime != null && endTime == null) { + dir = SortDirection.ASCENDING; + } + Key testKey = KeyFactory.createKey(TestEntity.KIND, testName); + + Filter typeFilter = FilterUtil.getTestTypeFilter(showPresubmit, showPostsubmit, unfiltered); + Filter testFilter = + FilterUtil.getTimeFilter( + testKey, TestRunEntity.KIND, startTime, endTime, typeFilter); + + Map<String, String[]> parameterMap = request.getParameterMap(); + List<Filter> userTestFilters = FilterUtil.getUserTestFilters(parameterMap); + userTestFilters.add(0, testFilter); + Filter userDeviceFilter = FilterUtil.getUserDeviceFilter(parameterMap); + + List<TestRunMetadata> testRunMetadata = new ArrayList<>(); + Map<Key, TestRunMetadata> metadataMap = new HashMap<>(); + Key minKey = null; + Key maxKey = null; + List<Key> gets = + FilterUtil.getMatchingKeys( + testKey, + TestRunEntity.KIND, + userTestFilters, + userDeviceFilter, + dir, + MAX_RESULT_COUNT); + Map<Key, Entity> entityMap = datastore.get(gets); + for (Key key : gets) { + if (!entityMap.containsKey(key)) { + continue; + } + TestRunEntity testRunEntity = TestRunEntity.fromEntity(entityMap.get(key)); + if (testRunEntity == null) { + continue; + } + if (minKey == null || key.compareTo(minKey) < 0) { + minKey = key; + } + if (maxKey == null || key.compareTo(maxKey) > 0) { + maxKey = key; + } + TestRunMetadata metadata = new TestRunMetadata(testName, testRunEntity); + testRunMetadata.add(metadata); + metadataMap.put(key, metadata); + } - SortDirection dir = SortDirection.DESCENDING; - if (startTime != null && endTime == null) { - dir = SortDirection.ASCENDING; + List<String> profilingPointNames = new ArrayList<>(); + if (minKey != null && maxKey != null) { + Filter deviceFilter = + FilterUtil.getDeviceTimeFilter( + testKey, TestRunEntity.KIND, minKey.getId(), maxKey.getId()); + Query deviceQuery = + new Query(DeviceInfoEntity.KIND) + .setAncestor(testKey) + .setFilter(deviceFilter) + .setKeysOnly(); + List<Key> deviceGets = new ArrayList<>(); + for (Entity device : + datastore + .prepare(deviceQuery) + .asIterable(DatastoreHelper.getLargeBatchOptions())) { + if (metadataMap.containsKey(device.getParent())) { + deviceGets.add(device.getKey()); } - Key testKey = KeyFactory.createKey(TestEntity.KIND, testName); - - Filter typeFilter = FilterUtil.getTestTypeFilter(showPresubmit, showPostsubmit, unfiltered); - Filter testFilter = - FilterUtil.getTimeFilter( - testKey, TestRunEntity.KIND, startTime, endTime, typeFilter); - - Map<String, String[]> parameterMap = request.getParameterMap(); - List<Filter> userTestFilters = FilterUtil.getUserTestFilters(parameterMap); - userTestFilters.add(0, testFilter); - Filter userDeviceFilter = FilterUtil.getUserDeviceFilter(parameterMap); - - List<TestRunMetadata> testRunMetadata = new ArrayList<>(); - Map<Key, TestRunMetadata> metadataMap = new HashMap<>(); - Key minKey = null; - Key maxKey = null; - List<Key> gets = - FilterUtil.getMatchingKeys( - testKey, - TestRunEntity.KIND, - userTestFilters, - userDeviceFilter, - dir, - MAX_RESULT_COUNT); - Map<Key, Entity> entityMap = datastore.get(gets); - for (Key key : gets) { - if (!entityMap.containsKey(key)) { - continue; - } - TestRunEntity testRunEntity = TestRunEntity.fromEntity(entityMap.get(key)); - if (testRunEntity == null) { - continue; - } - if (minKey == null || key.compareTo(minKey) < 0) { - minKey = key; - } - if (maxKey == null || key.compareTo(maxKey) > 0) { - maxKey = key; - } - TestRunMetadata metadata = new TestRunMetadata(testName, testRunEntity); - testRunMetadata.add(metadata); - metadataMap.put(key, metadata); + } + Map<Key, Entity> devices = datastore.get(deviceGets); + for (Key key : devices.keySet()) { + if (!metadataMap.containsKey(key.getParent())) { + continue; } - - List<String> profilingPointNames = new ArrayList<>(); - if (minKey != null && maxKey != null) { - Filter deviceFilter = - FilterUtil.getDeviceTimeFilter( - testKey, TestRunEntity.KIND, minKey.getId(), maxKey.getId()); - Query deviceQuery = - new Query(DeviceInfoEntity.KIND) - .setAncestor(testKey) - .setFilter(deviceFilter) - .setKeysOnly(); - List<Key> deviceGets = new ArrayList<>(); - for (Entity device : - datastore - .prepare(deviceQuery) - .asIterable(DatastoreHelper.getLargeBatchOptions())) { - if (metadataMap.containsKey(device.getParent())) { - deviceGets.add(device.getKey()); - } - } - Map<Key, Entity> devices = datastore.get(deviceGets); - for (Key key : devices.keySet()) { - if (!metadataMap.containsKey(key.getParent())) continue; - DeviceInfoEntity device = DeviceInfoEntity.fromEntity(devices.get(key)); - if (device == null) continue; - TestRunMetadata metadata = metadataMap.get(key.getParent()); - metadata.addDevice(device); - } - - Filter profilingFilter = - FilterUtil.getProfilingTimeFilter( - testKey, TestRunEntity.KIND, minKey.getId(), maxKey.getId()); - - Set<String> profilingPoints = new HashSet<>(); - Query profilingPointQuery = - new Query(ProfilingPointRunEntity.KIND) - .setAncestor(testKey) - .setFilter(profilingFilter) - .setKeysOnly(); - for (Entity e : datastore.prepare(profilingPointQuery).asIterable()) { - profilingPoints.add(e.getKey().getName()); - } - - if (profilingPoints.size() == 0) { - profilingDataAlert = PROFILING_DATA_ALERT; - } - profilingPointNames.addAll(profilingPoints); - profilingPointNames.sort(Comparator.naturalOrder()); + DeviceInfoEntity device = DeviceInfoEntity.fromEntity(devices.get(key)); + if (device == null) { + continue; } + TestRunMetadata metadata = metadataMap.get(key.getParent()); + metadata.addDevice(device); + } + + Filter profilingFilter = + FilterUtil.getProfilingTimeFilter( + testKey, TestRunEntity.KIND, minKey.getId(), maxKey.getId()); + + Set<String> profilingPoints = new HashSet<>(); + Query profilingPointQuery = + new Query(ProfilingPointRunEntity.KIND) + .setAncestor(testKey) + .setFilter(profilingFilter) + .setKeysOnly(); + for (Entity e : datastore.prepare(profilingPointQuery).asIterable()) { + profilingPoints.add(e.getKey().getName()); + } + + if (profilingPoints.size() == 0) { + profilingDataAlert = PROFILING_DATA_ALERT; + } + profilingPointNames.addAll(profilingPoints); + profilingPointNames.sort(Comparator.naturalOrder()); + } - testRunMetadata.sort( - (t1, t2) -> - new Long(t2.testRun.startTimestamp).compareTo(t1.testRun.startTimestamp)); - List<JsonObject> testRunObjects = new ArrayList<>(); - - int prefetchCount = 0; - for (TestRunMetadata metadata : testRunMetadata) { - if (metadata.testRun.failCount > 0 && prefetchCount < MAX_PREFETCH_COUNT) { - // process - metadata.addDetails(processTestDetails(metadata)); - ++prefetchCount; - } - testRunObjects.add(metadata.toJson()); - } + testRunMetadata.sort( + (t1, t2) -> + new Long(t2.testRun.getStartTimestamp()).compareTo(t1.testRun.getStartTimestamp())); + List<JsonObject> testRunObjects = new ArrayList<>(); + + int prefetchCount = 0; + for (TestRunMetadata metadata : testRunMetadata) { + if (metadata.testRun.getFailCount() > 0 && prefetchCount < MAX_PREFETCH_COUNT) { + // process + metadata.addDetails(processTestDetails(metadata)); + ++prefetchCount; + } + testRunObjects.add(metadata.toJson()); + } - int[] topBuildResultCounts = null; - String topBuild = ""; - if (testRunMetadata.size() > 0) { - TestRunMetadata firstRun = testRunMetadata.get(0); - topBuild = firstRun.getDeviceInfo(); - endTime = firstRun.testRun.startTimestamp; - TestRunDetails topDetails = firstRun.getDetails(); - if (topDetails == null) { - topDetails = processTestDetails(firstRun); - } - topBuildResultCounts = topDetails.resultCounts; - - TestRunMetadata lastRun = testRunMetadata.get(testRunMetadata.size() - 1); - startTime = lastRun.testRun.startTimestamp; - } + int[] topBuildResultCounts = null; + String topBuild = ""; + if (testRunMetadata.size() > 0) { + TestRunMetadata firstRun = testRunMetadata.get(0); + topBuild = firstRun.getDeviceInfo(); + endTime = firstRun.testRun.getStartTimestamp(); + TestRunDetails topDetails = firstRun.getDetails(); + if (topDetails == null) { + topDetails = processTestDetails(firstRun); + } + topBuildResultCounts = topDetails.resultCounts; + + TestRunMetadata lastRun = testRunMetadata.get(testRunMetadata.size() - 1); + startTime = lastRun.testRun.getStartTimestamp(); + } - FilterUtil.setAttributes(request, parameterMap); - - request.setAttribute("testName", request.getParameter("testName")); - - request.setAttribute("error", profilingDataAlert); - - request.setAttribute("profilingPointNames", profilingPointNames); - request.setAttribute("resultNames", resultNames); - request.setAttribute("resultNamesJson", new Gson().toJson(resultNames)); - request.setAttribute("testRuns", new Gson().toJson(testRunObjects)); - - // data for pie chart - request.setAttribute("topBuildResultCounts", new Gson().toJson(topBuildResultCounts)); - request.setAttribute("topBuildId", topBuild); - request.setAttribute("startTime", new Gson().toJson(startTime)); - request.setAttribute("endTime", new Gson().toJson(endTime)); - request.setAttribute( - "hasNewer", - new Gson().toJson(DatastoreHelper.hasNewer(testKey, TestRunEntity.KIND, endTime))); - request.setAttribute( - "hasOlder", - new Gson() - .toJson(DatastoreHelper.hasOlder(testKey, TestRunEntity.KIND, startTime))); - request.setAttribute("unfiltered", unfiltered); - request.setAttribute("showPresubmit", showPresubmit); - request.setAttribute("showPostsubmit", showPostsubmit); - - request.setAttribute("branches", new Gson().toJson(DatastoreHelper.getAllBranches())); - request.setAttribute("devices", new Gson().toJson(DatastoreHelper.getAllBuildFlavors())); - - dispatcher = request.getRequestDispatcher(TABLE_JSP); - try { - dispatcher.forward(request, response); - } catch (ServletException e) { - logger.log(Level.SEVERE, "Servlet Exception caught : " + e.toString()); - } + FilterUtil.setAttributes(request, parameterMap); + + request.setAttribute("testName", request.getParameter("testName")); + + request.setAttribute("error", profilingDataAlert); + + request.setAttribute("profilingPointNames", profilingPointNames); + request.setAttribute("resultNames", resultNames); + request.setAttribute("resultNamesJson", new Gson().toJson(resultNames)); + request.setAttribute("testRuns", new Gson().toJson(testRunObjects)); + + // data for pie chart + request.setAttribute("topBuildResultCounts", new Gson().toJson(topBuildResultCounts)); + request.setAttribute("topBuildId", topBuild); + request.setAttribute("startTime", new Gson().toJson(startTime)); + request.setAttribute("endTime", new Gson().toJson(endTime)); + request.setAttribute( + "hasNewer", + new Gson().toJson(DatastoreHelper.hasNewer(testKey, TestRunEntity.KIND, endTime))); + request.setAttribute( + "hasOlder", + new Gson() + .toJson(DatastoreHelper.hasOlder(testKey, TestRunEntity.KIND, startTime))); + request.setAttribute("unfiltered", unfiltered); + request.setAttribute("showPresubmit", showPresubmit); + request.setAttribute("showPostsubmit", showPostsubmit); + + request.setAttribute("branches", new Gson().toJson(DatastoreHelper.getAllBranches())); + request.setAttribute("devices", new Gson().toJson(DatastoreHelper.getAllBuildFlavors())); + + dispatcher = request.getRequestDispatcher(TABLE_JSP); + try { + dispatcher.forward(request, response); + } catch (ServletException e) { + logger.log(Level.SEVERE, "Servlet Exception caught : " + e.toString()); } + } } diff --git a/src/main/java/com/android/vts/util/DatastoreHelper.java b/src/main/java/com/android/vts/util/DatastoreHelper.java index 364ed04..c0ddf9d 100644 --- a/src/main/java/com/android/vts/util/DatastoreHelper.java +++ b/src/main/java/com/android/vts/util/DatastoreHelper.java @@ -63,501 +63,512 @@ import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; -/** DatastoreHelper, a helper class for interacting with Cloud Datastore. */ +/** + * DatastoreHelper, a helper class for interacting with Cloud Datastore. + */ public class DatastoreHelper { - /** The default kind name for datastore */ - public static final String NULL_ENTITY_KIND = "nullEntity"; - - public static final int MAX_WRITE_RETRIES = 5; - /** - * This variable is for maximum number of entities per transaction You can find the detail here - * (https://cloud.google.com/datastore/docs/concepts/limits) - */ - public static final int MAX_ENTITY_SIZE_PER_TRANSACTION = 300; - - protected static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName()); - private static final DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - - /** - * Get query fetch options for large batches of entities. - * - * @return FetchOptions with a large chunk and prefetch size. - */ - public static FetchOptions getLargeBatchOptions() { - return FetchOptions.Builder.withChunkSize(1000).prefetchSize(1000); - } - /** - * Returns true if there are data points newer than lowerBound in the results table. - * - * @param parentKey The parent key to use in the query. - * @param kind The query entity kind. - * @param lowerBound The (exclusive) lower time bound, long, microseconds. - * @return boolean True if there are newer data points. - * @throws IOException - */ - public static boolean hasNewer(Key parentKey, String kind, Long lowerBound) { - if (lowerBound == null || lowerBound <= 0) return false; - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - Key startKey = KeyFactory.createKey(parentKey, kind, lowerBound); - Filter startFilter = - new FilterPredicate( - Entity.KEY_RESERVED_PROPERTY, FilterOperator.GREATER_THAN, startKey); - Query q = new Query(kind).setAncestor(parentKey).setFilter(startFilter).setKeysOnly(); - return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0; + /** + * The default kind name for datastore + */ + public static final String NULL_ENTITY_KIND = "nullEntity"; + + public static final int MAX_WRITE_RETRIES = 5; + /** + * This variable is for maximum number of entities per transaction You can find the detail here + * (https://cloud.google.com/datastore/docs/concepts/limits) + */ + public static final int MAX_ENTITY_SIZE_PER_TRANSACTION = 300; + + protected static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName()); + private static final DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); + + /** + * Get query fetch options for large batches of entities. + * + * @return FetchOptions with a large chunk and prefetch size. + */ + public static FetchOptions getLargeBatchOptions() { + return FetchOptions.Builder.withChunkSize(1000).prefetchSize(1000); + } + + /** + * Returns true if there are data points newer than lowerBound in the results table. + * + * @param parentKey The parent key to use in the query. + * @param kind The query entity kind. + * @param lowerBound The (exclusive) lower time bound, long, microseconds. + * @return boolean True if there are newer data points. + */ + public static boolean hasNewer(Key parentKey, String kind, Long lowerBound) { + if (lowerBound == null || lowerBound <= 0) { + return false; } - - /** - * Returns true if there are data points older than upperBound in the table. - * - * @param parentKey The parent key to use in the query. - * @param kind The query entity kind. - * @param upperBound The (exclusive) upper time bound, long, microseconds. - * @return boolean True if there are older data points. - * @throws IOException - */ - public static boolean hasOlder(Key parentKey, String kind, Long upperBound) { - if (upperBound == null || upperBound <= 0) return false; - Key endKey = KeyFactory.createKey(parentKey, kind, upperBound); - Filter endFilter = - new FilterPredicate(Entity.KEY_RESERVED_PROPERTY, FilterOperator.LESS_THAN, endKey); - Query q = new Query(kind).setAncestor(parentKey).setFilter(endFilter).setKeysOnly(); - return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0; + DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); + Key startKey = KeyFactory.createKey(parentKey, kind, lowerBound); + Filter startFilter = + new FilterPredicate( + Entity.KEY_RESERVED_PROPERTY, FilterOperator.GREATER_THAN, startKey); + Query q = new Query(kind).setAncestor(parentKey).setFilter(startFilter).setKeysOnly(); + return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0; + } + + /** + * Returns true if there are data points older than upperBound in the table. + * + * @param parentKey The parent key to use in the query. + * @param kind The query entity kind. + * @param upperBound The (exclusive) upper time bound, long, microseconds. + * @return boolean True if there are older data points. + */ + public static boolean hasOlder(Key parentKey, String kind, Long upperBound) { + if (upperBound == null || upperBound <= 0) { + return false; } - - /** - * Get all of the devices branches. - * - * @return a list of all branches. - */ - public static List<String> getAllBranches() { - Query query = new Query(BranchEntity.KIND).setKeysOnly(); - List<String> branches = new ArrayList<>(); - for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) { - branches.add(e.getKey().getName()); - } - return branches; + Key endKey = KeyFactory.createKey(parentKey, kind, upperBound); + Filter endFilter = + new FilterPredicate(Entity.KEY_RESERVED_PROPERTY, FilterOperator.LESS_THAN, endKey); + Query q = new Query(kind).setAncestor(parentKey).setFilter(endFilter).setKeysOnly(); + return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0; + } + + /** + * Get all of the devices branches. + * + * @return a list of all branches. + */ + public static List<String> getAllBranches() { + Query query = new Query(BranchEntity.KIND).setKeysOnly(); + List<String> branches = new ArrayList<>(); + for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) { + branches.add(e.getKey().getName()); } - - /** - * Get all of the device build flavors. - * - * @return a list of all device build flavors. - */ - public static List<String> getAllBuildFlavors() { - Query query = new Query(BuildTargetEntity.KIND).setKeysOnly(); - List<String> devices = new ArrayList<>(); - for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) { - devices.add(e.getKey().getName()); - } - return devices; + return branches; + } + + /** + * Get all of the device build flavors. + * + * @return a list of all device build flavors. + */ + public static List<String> getAllBuildFlavors() { + Query query = new Query(BuildTargetEntity.KIND).setKeysOnly(); + List<String> devices = new ArrayList<>(); + for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) { + devices.add(e.getKey().getName()); } - - /** - * Upload data from a test report message - * - * @param report The test report containing data to upload. - */ - public static void insertTestReport(TestReportMessage report) { - - List<Entity> testEntityList = new ArrayList<>(); - List<Entity> branchEntityList = new ArrayList<>(); - List<Entity> buildTargetEntityList = new ArrayList<>(); - List<Entity> coverageEntityList = new ArrayList<>(); - List<Entity> profilingPointRunEntityList = new ArrayList<>(); - - if (!report.hasStartTimestamp() - || !report.hasEndTimestamp() - || !report.hasTest() - || !report.hasHostInfo() - || !report.hasBuildInfo()) { - // missing information - return; - } - long startTimestamp = report.getStartTimestamp(); - long endTimestamp = report.getEndTimestamp(); - String testName = report.getTest().toStringUtf8(); - String testBuildId = report.getBuildInfo().getId().toStringUtf8(); - String hostName = report.getHostInfo().getHostname().toStringUtf8(); - - TestEntity testEntity = new TestEntity(testName); - - Key testRunKey = - KeyFactory.createKey( - testEntity.key, TestRunEntity.KIND, report.getStartTimestamp()); - - long passCount = 0; - long failCount = 0; - long coveredLineCount = 0; - long totalLineCount = 0; - - Set<Key> buildTargetKeys = new HashSet<>(); - Set<Key> branchKeys = new HashSet<>(); - List<TestCaseRunEntity> testCases = new ArrayList<>(); - List<Key> profilingPointKeys = new ArrayList<>(); - List<String> links = new ArrayList<>(); - - // Process test cases - for (TestCaseReportMessage testCase : report.getTestCaseList()) { - String testCaseName = testCase.getName().toStringUtf8(); - TestCaseResult result = testCase.getTestResult(); - // Track global pass/fail counts - if (result == TestCaseResult.TEST_CASE_RESULT_PASS) { - ++passCount; - } else if (result != TestCaseResult.TEST_CASE_RESULT_SKIP) { - ++failCount; - } - if (testCase.getSystraceCount() > 0 - && testCase.getSystraceList().get(0).getUrlCount() > 0) { - String systraceLink = testCase.getSystraceList().get(0).getUrl(0).toStringUtf8(); - links.add(systraceLink); - } - // Process coverage data for test case - for (CoverageReportMessage coverage : testCase.getCoverageList()) { - CoverageEntity coverageEntity = - CoverageEntity.fromCoverageReport(testRunKey, testCaseName, coverage); - if (coverageEntity == null) { - logger.log(Level.WARNING, "Invalid coverage report in test run " + testRunKey); - } else { - coveredLineCount += coverageEntity.coveredLineCount; - totalLineCount += coverageEntity.totalLineCount; - coverageEntityList.add(coverageEntity.toEntity()); - } - } - // Process profiling data for test case - for (ProfilingReportMessage profiling : testCase.getProfilingList()) { - ProfilingPointRunEntity profilingPointRunEntity = - ProfilingPointRunEntity.fromProfilingReport(testRunKey, profiling); - if (profilingPointRunEntity == null) { - logger.log(Level.WARNING, "Invalid profiling report in test run " + testRunKey); - } else { - profilingPointRunEntityList.add(profilingPointRunEntity.toEntity()); - profilingPointKeys.add(profilingPointRunEntity.key); - testEntity.setHasProfilingData(true); - } - } - - int lastIndex = testCases.size() - 1; - if (lastIndex < 0 || testCases.get(lastIndex).isFull()) { - testCases.add(new TestCaseRunEntity()); - ++lastIndex; - } - TestCaseRunEntity testCaseEntity = testCases.get(lastIndex); - testCaseEntity.addTestCase(testCaseName, result.getNumber()); + return devices; + } + + /** + * Upload data from a test report message + * + * @param report The test report containing data to upload. + */ + public static void insertTestReport(TestReportMessage report) { + + List<Entity> testEntityList = new ArrayList<>(); + List<Entity> branchEntityList = new ArrayList<>(); + List<Entity> buildTargetEntityList = new ArrayList<>(); + List<Entity> coverageEntityList = new ArrayList<>(); + List<Entity> profilingPointRunEntityList = new ArrayList<>(); + + if (!report.hasStartTimestamp() + || !report.hasEndTimestamp() + || !report.hasTest() + || !report.hasHostInfo() + || !report.hasBuildInfo()) { + // missing information + return; + } + long startTimestamp = report.getStartTimestamp(); + long endTimestamp = report.getEndTimestamp(); + String testName = report.getTest().toStringUtf8(); + String testBuildId = report.getBuildInfo().getId().toStringUtf8(); + String hostName = report.getHostInfo().getHostname().toStringUtf8(); + + TestEntity testEntity = new TestEntity(testName); + + Key testRunKey = + KeyFactory.createKey( + testEntity.getOldKey(), TestRunEntity.KIND, report.getStartTimestamp()); + + long passCount = 0; + long failCount = 0; + long coveredLineCount = 0; + long totalLineCount = 0; + + Set<Key> buildTargetKeys = new HashSet<>(); + Set<Key> branchKeys = new HashSet<>(); + List<TestCaseRunEntity> testCases = new ArrayList<>(); + List<Key> profilingPointKeys = new ArrayList<>(); + List<String> links = new ArrayList<>(); + + // Process test cases + for (TestCaseReportMessage testCase : report.getTestCaseList()) { + String testCaseName = testCase.getName().toStringUtf8(); + TestCaseResult result = testCase.getTestResult(); + // Track global pass/fail counts + if (result == TestCaseResult.TEST_CASE_RESULT_PASS) { + ++passCount; + } else if (result != TestCaseResult.TEST_CASE_RESULT_SKIP) { + ++failCount; + } + if (testCase.getSystraceCount() > 0 + && testCase.getSystraceList().get(0).getUrlCount() > 0) { + String systraceLink = testCase.getSystraceList().get(0).getUrl(0).toStringUtf8(); + links.add(systraceLink); + } + // Process coverage data for test case + for (CoverageReportMessage coverage : testCase.getCoverageList()) { + CoverageEntity coverageEntity = + CoverageEntity.fromCoverageReport(testRunKey, testCaseName, coverage); + if (coverageEntity == null) { + logger.log(Level.WARNING, "Invalid coverage report in test run " + testRunKey); + } else { + coveredLineCount += coverageEntity.getCoveredCount(); + totalLineCount += coverageEntity.getTotalCount(); + coverageEntityList.add(coverageEntity.toEntity()); } - - List<Entity> testCasePuts = new ArrayList<>(); - for (TestCaseRunEntity testCaseEntity : testCases) { - testCasePuts.add(testCaseEntity.toEntity()); + } + // Process profiling data for test case + for (ProfilingReportMessage profiling : testCase.getProfilingList()) { + ProfilingPointRunEntity profilingPointRunEntity = + ProfilingPointRunEntity.fromProfilingReport(testRunKey, profiling); + if (profilingPointRunEntity == null) { + logger.log(Level.WARNING, "Invalid profiling report in test run " + testRunKey); + } else { + profilingPointRunEntityList.add(profilingPointRunEntity.toEntity()); + profilingPointKeys.add(profilingPointRunEntity.key); + testEntity.setHasProfilingData(true); } - List<Key> testCaseKeys = datastore.put(testCasePuts); + } + + int lastIndex = testCases.size() - 1; + if (lastIndex < 0 || testCases.get(lastIndex).isFull()) { + testCases.add(new TestCaseRunEntity()); + ++lastIndex; + } + TestCaseRunEntity testCaseEntity = testCases.get(lastIndex); + testCaseEntity.addTestCase(testCaseName, result.getNumber()); + } - List<Long> testCaseIds = new ArrayList<>(); - for (Key key : testCaseKeys) { - testCaseIds.add(key.getId()); - } + List<Entity> testCasePuts = new ArrayList<>(); + for (TestCaseRunEntity testCaseEntity : testCases) { + testCasePuts.add(testCaseEntity.toEntity()); + } + List<Key> testCaseKeys = datastore.put(testCasePuts); - // Process device information - TestRunType testRunType = null; - for (AndroidDeviceInfoMessage device : report.getDeviceInfoList()) { - DeviceInfoEntity deviceInfoEntity = - DeviceInfoEntity.fromDeviceInfoMessage(testRunKey, device); - if (deviceInfoEntity == null) { - logger.log(Level.WARNING, "Invalid device info in test run " + testRunKey); - } else { - // Run type on devices must be the same, else set to OTHER - TestRunType runType = TestRunType.fromBuildId(deviceInfoEntity.buildId); - if (testRunType == null) { - testRunType = runType; - } else if (runType != testRunType) { - testRunType = TestRunType.OTHER; - } - testEntityList.add(deviceInfoEntity.toEntity()); - BuildTargetEntity target = new BuildTargetEntity(deviceInfoEntity.buildFlavor); - if (buildTargetKeys.add(target.key)) { - buildTargetEntityList.add(target.toEntity()); - } - BranchEntity branch = new BranchEntity(deviceInfoEntity.branch); - if (branchKeys.add(branch.key)) { - branchEntityList.add(branch.toEntity()); - } - } - } + List<Long> testCaseIds = new ArrayList<>(); + for (Key key : testCaseKeys) { + testCaseIds.add(key.getId()); + } - // Overall run type should be determined by the device builds unless test build is OTHER + // Process device information + TestRunType testRunType = null; + for (AndroidDeviceInfoMessage device : report.getDeviceInfoList()) { + DeviceInfoEntity deviceInfoEntity = + DeviceInfoEntity.fromDeviceInfoMessage(testRunKey, device); + if (deviceInfoEntity == null) { + logger.log(Level.WARNING, "Invalid device info in test run " + testRunKey); + } else { + // Run type on devices must be the same, else set to OTHER + TestRunType runType = TestRunType.fromBuildId(deviceInfoEntity.buildId); if (testRunType == null) { - testRunType = TestRunType.fromBuildId(testBuildId); - } else if (TestRunType.fromBuildId(testBuildId) == TestRunType.OTHER) { - testRunType = TestRunType.OTHER; + testRunType = runType; + } else if (runType != testRunType) { + testRunType = TestRunType.OTHER; } - - // Process global coverage data - for (CoverageReportMessage coverage : report.getCoverageList()) { - CoverageEntity coverageEntity = - CoverageEntity.fromCoverageReport(testRunKey, new String(), coverage); - if (coverageEntity == null) { - logger.log(Level.WARNING, "Invalid coverage report in test run " + testRunKey); - } else { - coveredLineCount += coverageEntity.coveredLineCount; - totalLineCount += coverageEntity.totalLineCount; - coverageEntityList.add(coverageEntity.toEntity()); - } + testEntityList.add(deviceInfoEntity.toEntity()); + BuildTargetEntity target = new BuildTargetEntity(deviceInfoEntity.buildFlavor); + if (buildTargetKeys.add(target.key)) { + buildTargetEntityList.add(target.toEntity()); } - - // Process global profiling data - for (ProfilingReportMessage profiling : report.getProfilingList()) { - ProfilingPointRunEntity profilingPointRunEntity = - ProfilingPointRunEntity.fromProfilingReport(testRunKey, profiling); - if (profilingPointRunEntity == null) { - logger.log(Level.WARNING, "Invalid profiling report in test run " + testRunKey); - } else { - profilingPointRunEntityList.add(profilingPointRunEntity.toEntity()); - profilingPointKeys.add(profilingPointRunEntity.key); - testEntity.setHasProfilingData(true); - } + BranchEntity branch = new BranchEntity(deviceInfoEntity.branch); + if (branchKeys.add(branch.key)) { + branchEntityList.add(branch.toEntity()); } + } + } - // Process log data - for (LogMessage log : report.getLogList()) { - if (log.hasUrl()) links.add(log.getUrl().toStringUtf8()); - } - // Process url resource - for (UrlResourceMessage resource : report.getLinkResourceList()) { - if (resource.hasUrl()) links.add(resource.getUrl().toStringUtf8()); - } + // Overall run type should be determined by the device builds unless test build is OTHER + if (testRunType == null) { + testRunType = TestRunType.fromBuildId(testBuildId); + } else if (TestRunType.fromBuildId(testBuildId) == TestRunType.OTHER) { + testRunType = TestRunType.OTHER; + } - TestRunEntity testRunEntity = - new TestRunEntity( - testEntity.key, - testRunType, - startTimestamp, - endTimestamp, - testBuildId, - hostName, - passCount, - failCount, - testCaseIds, - links, - coveredLineCount, - totalLineCount); - testEntityList.add(testRunEntity.toEntity()); - - Entity test = testEntity.toEntity(); - - if (datastoreTransactionalRetry(test, testEntityList)) { - List<List<Entity>> auxiliaryEntityList = - Arrays.asList( - profilingPointRunEntityList, - coverageEntityList, - branchEntityList, - buildTargetEntityList); - int indexCount = 0; - for (List<Entity> entityList : auxiliaryEntityList) { - switch (indexCount) { - case 0: - case 1: - if (entityList.size() > MAX_ENTITY_SIZE_PER_TRANSACTION) { - List<List<Entity>> partitionedList = - Lists.partition(entityList, MAX_ENTITY_SIZE_PER_TRANSACTION); - partitionedList.forEach( - subEntityList -> { - datastoreTransactionalRetry( - new Entity(NULL_ENTITY_KIND), subEntityList); - }); - } else { - datastoreTransactionalRetry(new Entity(NULL_ENTITY_KIND), entityList); - } - break; - case 2: - case 3: - datastoreTransactionalRetryWithXG( - new Entity(NULL_ENTITY_KIND), entityList, true); - break; - default: - break; - } - indexCount++; - } + // Process global coverage data + for (CoverageReportMessage coverage : report.getCoverageList()) { + CoverageEntity coverageEntity = + CoverageEntity.fromCoverageReport(testRunKey, new String(), coverage); + if (coverageEntity == null) { + logger.log(Level.WARNING, "Invalid coverage report in test run " + testRunKey); + } else { + coveredLineCount += coverageEntity.getCoveredCount(); + totalLineCount += coverageEntity.getTotalCount(); + coverageEntityList.add(coverageEntity.toEntity()); + } + } - if (testRunEntity.type == TestRunType.POSTSUBMIT) { - VtsAlertJobServlet.addTask(testRunKey); - if (testRunEntity.hasCoverage) { - VtsCoverageAlertJobServlet.addTask(testRunKey); - } - if (profilingPointKeys.size() > 0) { - VtsProfilingStatsJobServlet.addTasks(profilingPointKeys); - } - } else { - logger.log( - Level.WARNING, - "The alert email was not sent as testRunEntity type is not POSTSUBMIT!" + - " \n " + " testRunEntity type => " + testRunEntity.type); - } - } + // Process global profiling data + for (ProfilingReportMessage profiling : report.getProfilingList()) { + ProfilingPointRunEntity profilingPointRunEntity = + ProfilingPointRunEntity.fromProfilingReport(testRunKey, profiling); + if (profilingPointRunEntity == null) { + logger.log(Level.WARNING, "Invalid profiling report in test run " + testRunKey); + } else { + profilingPointRunEntityList.add(profilingPointRunEntity.toEntity()); + profilingPointKeys.add(profilingPointRunEntity.key); + testEntity.setHasProfilingData(true); + } } - /** - * Upload data from a test plan report message - * - * @param report The test plan report containing data to upload. - */ - public static void insertTestPlanReport(TestPlanReportMessage report) { - List<Entity> testEntityList = new ArrayList<>(); - - List<String> testModules = report.getTestModuleNameList(); - List<Long> testTimes = report.getTestModuleStartTimestampList(); - if (testModules.size() != testTimes.size() || !report.hasTestPlanName()) { - logger.log(Level.WARNING, "TestPlanReportMessage is missing information."); - return; - } + // Process log data + for (LogMessage log : report.getLogList()) { + if (log.hasUrl()) { + links.add(log.getUrl().toStringUtf8()); + } + } + // Process url resource + for (UrlResourceMessage resource : report.getLinkResourceList()) { + if (resource.hasUrl()) { + links.add(resource.getUrl().toStringUtf8()); + } + } - String testPlanName = report.getTestPlanName(); - Entity testPlanEntity = new TestPlanEntity(testPlanName).toEntity(); - List<Key> testRunKeys = new ArrayList<>(); - for (int i = 0; i < testModules.size(); i++) { - String test = testModules.get(i); - long time = testTimes.get(i); - Key parentKey = KeyFactory.createKey(TestEntity.KIND, test); - Key testRunKey = KeyFactory.createKey(parentKey, TestRunEntity.KIND, time); - testRunKeys.add(testRunKey); - } - Map<Key, Entity> testRuns = datastore.get(testRunKeys); - long passCount = 0; - long failCount = 0; - long startTimestamp = -1; - long endTimestamp = -1; - String testBuildId = null; - TestRunType type = null; - Set<DeviceInfoEntity> deviceInfoEntitySet = new HashSet<>(); - for (Key testRunKey : testRuns.keySet()) { - TestRunEntity testRun = TestRunEntity.fromEntity(testRuns.get(testRunKey)); - if (testRun == null) { - continue; // not a valid test run - } - passCount += testRun.passCount; - failCount += testRun.failCount; - if (startTimestamp < 0 || testRunKey.getId() < startTimestamp) { - startTimestamp = testRunKey.getId(); - } - if (endTimestamp < 0 || testRun.endTimestamp > endTimestamp) { - endTimestamp = testRun.endTimestamp; - } - if (type == null) { - type = testRun.type; - } else if (type != testRun.type) { - type = TestRunType.OTHER; - } - testBuildId = testRun.testBuildId; - Query deviceInfoQuery = new Query(DeviceInfoEntity.KIND).setAncestor(testRunKey); - for (Entity deviceInfoEntity : datastore.prepare(deviceInfoQuery).asIterable()) { - DeviceInfoEntity device = DeviceInfoEntity.fromEntity(deviceInfoEntity); - if (device == null) { - continue; // invalid entity - } - deviceInfoEntitySet.add(device); + TestRunEntity testRunEntity = + new TestRunEntity( + testEntity.getOldKey(), + testRunType, + startTimestamp, + endTimestamp, + testBuildId, + hostName, + passCount, + failCount, + testCaseIds, + links, + coveredLineCount, + totalLineCount); + testEntityList.add(testRunEntity.toEntity()); + + Entity test = testEntity.toEntity(); + + if (datastoreTransactionalRetry(test, testEntityList)) { + List<List<Entity>> auxiliaryEntityList = + Arrays.asList( + profilingPointRunEntityList, + coverageEntityList, + branchEntityList, + buildTargetEntityList); + int indexCount = 0; + for (List<Entity> entityList : auxiliaryEntityList) { + switch (indexCount) { + case 0: + case 1: + if (entityList.size() > MAX_ENTITY_SIZE_PER_TRANSACTION) { + List<List<Entity>> partitionedList = + Lists.partition(entityList, MAX_ENTITY_SIZE_PER_TRANSACTION); + partitionedList.forEach( + subEntityList -> { + datastoreTransactionalRetry( + new Entity(NULL_ENTITY_KIND), subEntityList); + }); + } else { + datastoreTransactionalRetry(new Entity(NULL_ENTITY_KIND), entityList); } + break; + case 2: + case 3: + datastoreTransactionalRetryWithXG( + new Entity(NULL_ENTITY_KIND), entityList, true); + break; + default: + break; } - if (startTimestamp < 0 || testBuildId == null || type == null) { - logger.log(Level.WARNING, "Couldn't infer test run information from runs."); - return; + indexCount++; + } + + if (testRunEntity.getType() == TestRunType.POSTSUBMIT) { + VtsAlertJobServlet.addTask(testRunKey); + if (testRunEntity.isHasCoverage()) { + VtsCoverageAlertJobServlet.addTask(testRunKey); } - TestPlanRunEntity testPlanRun = - new TestPlanRunEntity( - testPlanEntity.getKey(), - testPlanName, - type, - startTimestamp, - endTimestamp, - testBuildId, - passCount, - failCount, - testRunKeys); - - // Create the device infos. - for (DeviceInfoEntity device : deviceInfoEntitySet) { - testEntityList.add(device.copyWithParent(testPlanRun.key).toEntity()); + if (profilingPointKeys.size() > 0) { + VtsProfilingStatsJobServlet.addTasks(profilingPointKeys); } - testEntityList.add(testPlanRun.toEntity()); - - datastoreTransactionalRetry(testPlanEntity, testEntityList); + } else { + logger.log( + Level.WARNING, + "The alert email was not sent as testRunEntity type is not POSTSUBMIT!" + + " \n " + " testRunEntity type => " + testRunEntity.getType()); + } } - - /** - * Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times and withXG of - * false value - * - * @param entity The entity that you want to insert to datastore. - * @param entityList The list of entity for using datastore put method. - */ - private static boolean datastoreTransactionalRetry(Entity entity, List<Entity> entityList) { - return datastoreTransactionalRetryWithXG(entity, entityList, false); + } + + /** + * Upload data from a test plan report message + * + * @param report The test plan report containing data to upload. + */ + public static void insertTestPlanReport(TestPlanReportMessage report) { + List<Entity> testEntityList = new ArrayList<>(); + + List<String> testModules = report.getTestModuleNameList(); + List<Long> testTimes = report.getTestModuleStartTimestampList(); + if (testModules.size() != testTimes.size() || !report.hasTestPlanName()) { + logger.log(Level.WARNING, "TestPlanReportMessage is missing information."); + return; } - /** - * Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times - * - * @param entity The entity that you want to insert to datastore. - * @param entityList The list of entity for using datastore put method. - */ - private static boolean datastoreTransactionalRetryWithXG( - Entity entity, List<Entity> entityList, boolean withXG) { - int retries = 0; - while (true) { - Transaction txn; - if (withXG) { - TransactionOptions options = TransactionOptions.Builder.withXG(withXG); - txn = datastore.beginTransaction(options); + String testPlanName = report.getTestPlanName(); + Entity testPlanEntity = new TestPlanEntity(testPlanName).toEntity(); + List<Key> testRunKeys = new ArrayList<>(); + for (int i = 0; i < testModules.size(); i++) { + String test = testModules.get(i); + long time = testTimes.get(i); + Key parentKey = KeyFactory.createKey(TestEntity.KIND, test); + Key testRunKey = KeyFactory.createKey(parentKey, TestRunEntity.KIND, time); + testRunKeys.add(testRunKey); + } + Map<Key, Entity> testRuns = datastore.get(testRunKeys); + long passCount = 0; + long failCount = 0; + long startTimestamp = -1; + long endTimestamp = -1; + String testBuildId = null; + TestRunType type = null; + Set<DeviceInfoEntity> deviceInfoEntitySet = new HashSet<>(); + for (Key testRunKey : testRuns.keySet()) { + TestRunEntity testRun = TestRunEntity.fromEntity(testRuns.get(testRunKey)); + if (testRun == null) { + continue; // not a valid test run + } + passCount += testRun.getPassCount(); + failCount += testRun.getFailCount(); + if (startTimestamp < 0 || testRunKey.getId() < startTimestamp) { + startTimestamp = testRunKey.getId(); + } + if (endTimestamp < 0 || testRun.getEndTimestamp() > endTimestamp) { + endTimestamp = testRun.getEndTimestamp(); + } + if (type == null) { + type = testRun.getType(); + } else if (type != testRun.getType()) { + type = TestRunType.OTHER; + } + testBuildId = testRun.getTestBuildId(); + Query deviceInfoQuery = new Query(DeviceInfoEntity.KIND).setAncestor(testRunKey); + for (Entity deviceInfoEntity : datastore.prepare(deviceInfoQuery).asIterable()) { + DeviceInfoEntity device = DeviceInfoEntity.fromEntity(deviceInfoEntity); + if (device == null) { + continue; // invalid entity + } + deviceInfoEntitySet.add(device); + } + } + if (startTimestamp < 0 || testBuildId == null || type == null) { + logger.log(Level.WARNING, "Couldn't infer test run information from runs."); + return; + } + TestPlanRunEntity testPlanRun = + new TestPlanRunEntity( + testPlanEntity.getKey(), + testPlanName, + type, + startTimestamp, + endTimestamp, + testBuildId, + passCount, + failCount, + testRunKeys); + + // Create the device infos. + for (DeviceInfoEntity device : deviceInfoEntitySet) { + testEntityList.add(device.copyWithParent(testPlanRun.key).toEntity()); + } + testEntityList.add(testPlanRun.toEntity()); + + datastoreTransactionalRetry(testPlanEntity, testEntityList); + } + + /** + * Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times and withXG of + * false value + * + * @param entity The entity that you want to insert to datastore. + * @param entityList The list of entity for using datastore put method. + */ + private static boolean datastoreTransactionalRetry(Entity entity, List<Entity> entityList) { + return datastoreTransactionalRetryWithXG(entity, entityList, false); + } + + /** + * Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times + * + * @param entity The entity that you want to insert to datastore. + * @param entityList The list of entity for using datastore put method. + */ + private static boolean datastoreTransactionalRetryWithXG( + Entity entity, List<Entity> entityList, boolean withXG) { + int retries = 0; + while (true) { + Transaction txn; + if (withXG) { + TransactionOptions options = TransactionOptions.Builder.withXG(withXG); + txn = datastore.beginTransaction(options); + } else { + txn = datastore.beginTransaction(); + } + + try { + // Check if test already exists in the database + if (!entity.getKind().equalsIgnoreCase(NULL_ENTITY_KIND)) { + try { + if (entity.getKind().equalsIgnoreCase("Test")) { + Entity datastoreEntity = datastore.get(entity.getKey()); + TestEntity datastoreTestEntity = TestEntity.fromEntity(datastoreEntity); + if (datastoreTestEntity == null + || !datastoreTestEntity.equals(entity)) { + entityList.add(entity); + } + } else if (entity.getKind().equalsIgnoreCase("TestPlan")) { + datastore.get(entity.getKey()); } else { - txn = datastore.beginTransaction(); - } - - try { - // Check if test already exists in the database - if (!entity.getKind().equalsIgnoreCase(NULL_ENTITY_KIND)) { - try { - if (entity.getKind().equalsIgnoreCase("Test")) { - Entity datastoreEntity = datastore.get(entity.getKey()); - TestEntity datastoreTestEntity = TestEntity.fromEntity(datastoreEntity); - if (datastoreTestEntity == null - || !datastoreTestEntity.equals(entity)) { - entityList.add(entity); - } - } else if (entity.getKind().equalsIgnoreCase("TestPlan")) { - datastore.get(entity.getKey()); - } else { - datastore.get(entity.getKey()); - } - } catch (EntityNotFoundException e) { - entityList.add(entity); - } - } - datastore.put(txn, entityList); - txn.commit(); - break; - } catch (ConcurrentModificationException - | DatastoreFailureException - | DatastoreTimeoutException e) { - entityList.remove(entity); - logger.log( - Level.WARNING, - "Retrying insert kind: " + entity.getKind() + " key: " + entity.getKey()); - if (retries++ >= MAX_WRITE_RETRIES) { - logger.log( - Level.SEVERE, - "Exceeded maximum retries kind: " - + entity.getKind() - + " key: " - + entity.getKey()); - return false; - } - } finally { - if (txn.isActive()) { - logger.log( - Level.WARNING, "Transaction rollback forced for : " + entity.getKind()); - txn.rollback(); - } + datastore.get(entity.getKey()); } + } catch (EntityNotFoundException e) { + entityList.add(entity); + } + } + datastore.put(txn, entityList); + txn.commit(); + break; + } catch (ConcurrentModificationException + | DatastoreFailureException + | DatastoreTimeoutException e) { + entityList.remove(entity); + logger.log( + Level.WARNING, + "Retrying insert kind: " + entity.getKind() + " key: " + entity.getKey()); + if (retries++ >= MAX_WRITE_RETRIES) { + logger.log( + Level.SEVERE, + "Exceeded maximum retries kind: " + + entity.getKind() + + " key: " + + entity.getKey()); + return false; + } + } finally { + if (txn.isActive()) { + logger.log( + Level.WARNING, "Transaction rollback forced for : " + entity.getKind()); + txn.rollback(); } - return true; + } } + return true; + } } diff --git a/src/main/java/com/android/vts/util/EmailHelper.java b/src/main/java/com/android/vts/util/EmailHelper.java index 11bbdf5..0bec72b 100644 --- a/src/main/java/com/android/vts/util/EmailHelper.java +++ b/src/main/java/com/android/vts/util/EmailHelper.java @@ -74,9 +74,9 @@ public class EmailHelper { } if (testRun != null) { - sb.append("VTS Build ID: " + testRun.testBuildId + "<br>"); - sb.append("Start Time: " + TimeUtil.getDateTimeString(testRun.startTimestamp)); - sb.append("<br>End Time: " + TimeUtil.getDateTimeString(testRun.endTimestamp)); + sb.append("VTS Build ID: " + testRun.getTestBuildId() + "<br>"); + sb.append("Start Time: " + TimeUtil.getDateTimeString(testRun.getStartTimestamp())); + sb.append("<br>End Time: " + TimeUtil.getDateTimeString(testRun.getEndTimestamp())); } sb.append( "<br><br>For details, visit the" diff --git a/src/main/java/com/android/vts/util/TestResults.java b/src/main/java/com/android/vts/util/TestResults.java index 5ace5ca..70422ab 100644 --- a/src/main/java/com/android/vts/util/TestResults.java +++ b/src/main/java/com/android/vts/util/TestResults.java @@ -111,11 +111,11 @@ public class TestResults { public void addTestRun(Entity testRun, Iterable<Entity> testCaseRuns) { TestRunEntity testRunEntity = TestRunEntity.fromEntity(testRun); if (testRunEntity == null) return; - if (testRunEntity.startTimestamp < startTime) { - startTime = testRunEntity.startTimestamp; + if (testRunEntity.getStartTimestamp() < startTime) { + startTime = testRunEntity.getStartTimestamp(); } - if (testRunEntity.endTimestamp > endTime) { - endTime = testRunEntity.endTimestamp; + if (testRunEntity.getStartTimestamp() > endTime) { + endTime = testRunEntity.getStartTimestamp(); } testRuns.add(testRunEntity); testCaseRunMap.put(testRun.getKey(), new ArrayList<TestCaseRunEntity>()); @@ -139,8 +139,8 @@ public class TestResults { if (testRuns.size() == 0) return; TestRunEntity mostRecentRun = testRuns.get(0); - List<TestCaseRunEntity> testCaseResults = testCaseRunMap.get(mostRecentRun.key); - List<DeviceInfoEntity> deviceInfos = deviceInfoMap.get(mostRecentRun.key); + List<TestCaseRunEntity> testCaseResults = testCaseRunMap.get(mostRecentRun.getKey()); + List<DeviceInfoEntity> deviceInfos = deviceInfoMap.get(mostRecentRun.getKey()); if (deviceInfos.size() > 0) { DeviceInfoEntity totDevice = deviceInfos.get(0); totBuildId = totDevice.buildId; @@ -221,7 +221,7 @@ public class TestResults { processDeviceInfos(); processProfilingPoints(); } - testRuns.sort((t1, t2) -> new Long(t2.startTimestamp).compareTo(t1.startTimestamp)); + testRuns.sort((t1, t2) -> new Long(t2.getStartTimestamp()).compareTo(t1.getStartTimestamp())); generateToTBreakdown(); headerRow = new String[testRuns.size() + 1]; @@ -253,7 +253,7 @@ public class TestResults { TestRunEntity testRun = testRuns.get(col); // Process the device information - List<DeviceInfoEntity> devices = deviceInfoMap.get(testRun.key); + List<DeviceInfoEntity> devices = deviceInfoMap.get(testRun.getKey()); List<String> buildIdList = new ArrayList<>(); List<String> buildAliasList = new ArrayList<>(); List<String> buildFlavorList = new ArrayList<>(); @@ -281,17 +281,17 @@ public class TestResults { String productVariant = StringUtils.join(productVariantList, ","); String buildIds = StringUtils.join(buildIdList, ","); String abiInfo = StringUtils.join(abiInfoList, ","); - String vtsBuildId = testRun.testBuildId; + String vtsBuildId = testRun.getTestBuildId(); int totalCount = 0; - int passCount = (int) testRun.passCount; - int nonpassCount = (int) testRun.failCount; + int passCount = (int) testRun.getPassCount(); + int nonpassCount = (int) testRun.getFailCount(); TestCaseResult aggregateStatus = TestCaseResult.UNKNOWN_RESULT; - long totalLineCount = testRun.totalLineCount; - long coveredLineCount = testRun.coveredLineCount; + long totalLineCount = testRun.getTotalLineCount(); + long coveredLineCount = testRun.getCoveredLineCount(); // Process test case results - for (TestCaseRunEntity testCaseEntity : testCaseRunMap.get(testRun.key)) { + for (TestCaseRunEntity testCaseEntity : testCaseRunMap.get(testRun.getKey())) { // Update the aggregated test run status totalCount += testCaseEntity.testCases.size(); for (TestCase testCase : testCaseEntity.testCases) { @@ -358,7 +358,7 @@ public class TestResults { + "<a href=\"/show_coverage?testName=" + testName + "&startTime=" - + testRun.startTimestamp + + testRun.getStartTimestamp() + "\" class=\"waves-effect waves-light btn red right inline-btn\">" + "<i class=\"material-icons inline-icon\">menu</i></a>"; coverageInfo = coveredLineCount + "/" + totalLineCount; @@ -372,8 +372,8 @@ public class TestResults { List<String[]> linkEntries = new ArrayList<>(); logInfoMap.put(Integer.toString(col), linkEntries); - if (testRun.links != null) { - for (String rawUrl : testRun.links) { + if (testRun.getLinks() != null) { + for (String rawUrl : testRun.getLinks()) { LinkDisplay validatedLink = UrlUtil.processUrl(rawUrl); if (validatedLink == null) { logger.log(Level.WARNING, "Invalid logging URL : " + rawUrl); @@ -399,7 +399,7 @@ public class TestResults { } String icon = "<div class='status-icon " + aggregateStatus.toString() + "'> </div>"; - String hostname = testRun.hostName; + String hostname = testRun.getHostName(); // Populate the header row headerRow[col + 1] = @@ -430,11 +430,11 @@ public class TestResults { summaryGrid[6][col + 1] = linkSummary; // Populate the test time info grid - timeGrid[0][col + 1] = Long.toString(testRun.startTimestamp); - timeGrid[1][col + 1] = Long.toString(testRun.endTimestamp); + timeGrid[0][col + 1] = Long.toString(testRun.getStartTimestamp()); + timeGrid[1][col + 1] = Long.toString(testRun.getEndTimestamp()); // Populate the test duration info grid - durationGrid[0][col + 1] = Long.toString(testRun.endTimestamp - testRun.startTimestamp); + durationGrid[0][col + 1] = Long.toString(testRun.getEndTimestamp() - testRun.getStartTimestamp()); } profilingPointNames = |