diff options
author | Young Gyu Park <younggyu@google.com> | 2018-09-06 19:03:19 +0900 |
---|---|---|
committer | Young Gyu Park <younggyu@google.com> | 2018-09-14 11:52:32 +0900 |
commit | 5ad922ee5f85054d7b1b8fef4b6fa3f152beb0b5 (patch) | |
tree | bad67410edb69e50518a8fbd357097c94ca4fe4c /src/main/java | |
parent | 64e472e20f250cfc1ed0dc82731d816e9d6a69ef (diff) | |
download | dashboard-5ad922ee5f85054d7b1b8fef4b6fa3f152beb0b5.tar.gz |
Bug fix and loading speed improvement.
Test: go/vts-web-staging
Bug: 113356010
Change-Id: I4b9976fa7eb35efa3d17f69e8bc5c2bfba11066d
Diffstat (limited to 'src/main/java')
14 files changed, 258 insertions, 249 deletions
diff --git a/src/main/java/com/android/vts/api/BaseApiServlet.java b/src/main/java/com/android/vts/api/BaseApiServlet.java index 54a7574..1ad0237 100644 --- a/src/main/java/com/android/vts/api/BaseApiServlet.java +++ b/src/main/java/com/android/vts/api/BaseApiServlet.java @@ -30,7 +30,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; /** - * REST endpoint for posting test suite data to the Dashboard. + * An abstract class to be subclassed to create API Servlet */ public class BaseApiServlet extends HttpServlet { diff --git a/src/main/java/com/android/vts/api/TestDataForDevServlet.java b/src/main/java/com/android/vts/api/TestDataForDevServlet.java index a265b81..7fc067f 100644 --- a/src/main/java/com/android/vts/api/TestDataForDevServlet.java +++ b/src/main/java/com/android/vts/api/TestDataForDevServlet.java @@ -18,6 +18,7 @@ package com.android.vts.api; import com.android.vts.entity.BranchEntity; import com.android.vts.entity.BuildTargetEntity; +import com.android.vts.entity.CodeCoverageEntity; import com.android.vts.entity.CoverageEntity; import com.android.vts.entity.DeviceInfoEntity; import com.android.vts.entity.ProfilingPointRunEntity; @@ -26,14 +27,10 @@ import com.android.vts.entity.TestEntity; import com.android.vts.entity.TestPlanEntity; import com.android.vts.entity.TestPlanRunEntity; import com.android.vts.entity.TestRunEntity; -import com.android.vts.entity.TestSuiteFileEntity; -import com.android.vts.entity.TestSuiteResultEntity; import com.android.vts.entity.TestStatusEntity; - -import com.android.vts.entity.TestRunEntity.TestRunType; import com.android.vts.entity.TestStatusEntity.TestCaseReference; -import com.android.vts.servlet.BaseServlet; -import com.android.vts.util.EmailHelper; +import com.android.vts.entity.TestSuiteFileEntity; +import com.android.vts.entity.TestSuiteResultEntity; import com.google.appengine.api.datastore.DatastoreFailureException; import com.google.appengine.api.datastore.DatastoreService; import com.google.appengine.api.datastore.DatastoreServiceFactory; @@ -280,7 +277,7 @@ public class TestDataForDevServlet extends HttpServlet { .toString()); com.googlecode.objectify.Key< - TestSuiteFileEntity> + TestSuiteFileEntity> testSuiteFileParent = com.googlecode.objectify .Key.create( @@ -503,21 +500,30 @@ public class TestDataForDevServlet extends HttpServlet { datastore.put(branchEntity.toEntity()); }); + boolean hasCodeCoverage = + testRun.totalLineCount > 0 + && testRun.coveredLineCount >= 0; TestRunEntity testRunEntity = new TestRunEntity( testEntity.getOldKey(), - TestRunType.fromNumber(testRun.type), + testRun.type, testRun.startTimestamp, testRun.endTimestamp, testRun.testBuildId, testRun.hostName, testRun.passCount, testRun.failCount, + hasCodeCoverage, testRun.testCaseIds, - testRun.links, + testRun.links); + datastore.put(testRunEntity.toEntity()); + + CodeCoverageEntity codeCoverageEntity = + new CodeCoverageEntity( + testRunEntity.getKey(), testRun.coveredLineCount, testRun.totalLineCount); - datastore.put(testRunEntity.toEntity()); + datastore.put(codeCoverageEntity.toEntity()); Entity newTestEntity = testEntity.toEntity(); @@ -525,7 +531,8 @@ public class TestDataForDevServlet extends HttpServlet { try { // Check if test already exists in the datastore try { - Entity oldTest = datastore.get(testEntity.getOldKey()); + Entity oldTest = + datastore.get(testEntity.getOldKey()); TestEntity oldTestEntity = TestEntity.fromEntity(oldTest); if (oldTestEntity == null @@ -577,7 +584,7 @@ public class TestDataForDevServlet extends HttpServlet { long startTimestamp = -1; long endTimestamp = -1; String testBuildId = null; - TestRunType type = null; + long type = 0; Set<DeviceInfoEntity> devices = new HashSet<>(); for (Key testRunKey : testRuns.keySet()) { TestRunEntity testRun = @@ -593,11 +600,7 @@ public class TestDataForDevServlet extends HttpServlet { if (endTimestamp < 0 || testRun.getEndTimestamp() > endTimestamp) { endTimestamp = testRun.getEndTimestamp(); } - if (type == null) { - type = testRun.getType(); - } else if (type != testRun.getType()) { - type = TestRunType.OTHER; - } + type = testRun.getType(); testBuildId = testRun.getTestBuildId(); Query deviceInfoQuery = new Query(DeviceInfoEntity.KIND).setAncestor(testRunKey); @@ -611,7 +614,7 @@ public class TestDataForDevServlet extends HttpServlet { devices.add(device); } } - if (startTimestamp < 0 || testBuildId == null || type == null) { + if (startTimestamp < 0 || testBuildId == null || type == 0) { logger.log( Level.WARNING, "Couldn't infer test run information from runs."); diff --git a/src/main/java/com/android/vts/api/TestRunRestServlet.java b/src/main/java/com/android/vts/api/TestRunRestServlet.java index 67fd684..2c724ac 100644 --- a/src/main/java/com/android/vts/api/TestRunRestServlet.java +++ b/src/main/java/com/android/vts/api/TestRunRestServlet.java @@ -16,6 +16,7 @@ package com.android.vts.api; +import com.android.vts.entity.CodeCoverageEntity; import com.android.vts.entity.TestCaseRunEntity; import com.android.vts.entity.TestEntity; import com.android.vts.entity.TestRunEntity; @@ -35,13 +36,16 @@ import java.io.PrintWriter; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.logging.Logger; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import static com.googlecode.objectify.ObjectifyService.ofy; + /** Servlet for handling requests to fetch test case results. */ -public class TestRunRestServlet extends HttpServlet { +public class TestRunRestServlet extends BaseApiServlet { private static final String LATEST = "latest"; protected static final Logger logger = Logger.getLogger(TestRunRestServlet.class.getName()); @@ -52,8 +56,7 @@ public class TestRunRestServlet extends HttpServlet { * @param timeString The string representation of the test run timestamp (in microseconds). * @return A TestRunDetails object with the test case details for the specified run. */ - private static TestRunDetails getTestRunDetails(String test, String timeString) { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); + private TestRunDetails getTestRunDetails(String test, String timeString) { long timestamp; try { timestamp = Long.parseLong(timeString); @@ -63,68 +66,52 @@ public class TestRunRestServlet extends HttpServlet { return null; } - Key testKey = KeyFactory.createKey(TestEntity.KIND, test); - Key testRunKey = KeyFactory.createKey(testKey, TestRunEntity.KIND, timestamp); - TestRunEntity testRunEntity; - try { - Entity testRun = datastore.get(testRunKey); - testRunEntity = TestRunEntity.fromEntity(testRun); - if (testRunEntity == null) { - throw new EntityNotFoundException(testRunKey); - } - } catch (EntityNotFoundException e) { - return null; - } - TestRunDetails details = new TestRunDetails(); - List<Key> gets = new ArrayList<>(); - for (long testCaseId : testRunEntity.getTestCaseIds()) { - gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); - } - Map<Key, Entity> entityMap = datastore.get(gets); - for (Key key : entityMap.keySet()) { - TestCaseRunEntity testCaseRun = TestCaseRunEntity.fromEntity(entityMap.get(key)); - if (testCaseRun == null) { - continue; - } - details.addTestCase(testCaseRun); - } - return details; + TestRunEntity testRunEntity = TestRunEntity.getByTestNameId(test, timestamp); + + return getTestRunDetails(testRunEntity); } /** * Get the test case results for the latest run of the specified test. * - * @param test The test whose test cases to get. + * @param testName The test whose test cases to get. * @return A TestRunDetails object with the test case details for the latest run. */ - private static TestRunDetails getLatestTestRunDetails(String test) { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - Key testKey = KeyFactory.createKey(TestEntity.KIND, test); - Query.Filter typeFilter = FilterUtil.getTestTypeFilter(false, true, false); - Query testRunQuery = - new Query(TestRunEntity.KIND) - .setAncestor(testKey) - .setFilter(typeFilter) - .addSort(Entity.KEY_RESERVED_PROPERTY, Query.SortDirection.DESCENDING); - TestRunEntity testRun = null; - for (Entity testRunEntity : - datastore.prepare(testRunQuery).asIterable(FetchOptions.Builder.withLimit(1))) { - testRun = TestRunEntity.fromEntity(testRunEntity); - } + private TestRunDetails getLatestTestRunDetails(String testName) { + com.googlecode.objectify.Key testKey = + com.googlecode.objectify.Key.create( + TestEntity.class, testName); + + TestRunEntity testRun = ofy().load().type(TestRunEntity.class).ancestor(testKey) + .filter("type", 2).orderKey(true).first().now(); + if (testRun == null) return null; - TestRunDetails details = new TestRunDetails(); - List<Key> gets = new ArrayList<>(); - for (long testCaseId : testRun.getTestCaseIds()) { - gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); - } - Map<Key, Entity> entityMap = datastore.get(gets); - for (Key key : entityMap.keySet()) { - TestCaseRunEntity testCaseRun = TestCaseRunEntity.fromEntity(entityMap.get(key)); - if (testCaseRun == null) { - continue; + return getTestRunDetails(testRun); + } + + /** + * Get TestRunDetails instance from codeCoverageEntity instance. + * + * @param testRunEntity The TestRunEntity to access testCaseId. + * @return A TestRunDetails object with the test case details for the latest run. + */ + private TestRunDetails getTestRunDetails(TestRunEntity testRunEntity) { + TestRunDetails details = new TestRunDetails(); + List<com.googlecode.objectify.Key<TestCaseRunEntity>> testCaseKeyList = new ArrayList<>(); + if ( Objects.isNull(testRunEntity.getTestCaseIds()) ) { + return details; + } else { + for (long testCaseId : testRunEntity.getTestCaseIds()) { + testCaseKeyList.add( + com.googlecode.objectify.Key.create(TestCaseRunEntity.class, testCaseId)); + } + Map<com.googlecode.objectify.Key<TestCaseRunEntity>, TestCaseRunEntity> + testCaseRunEntityKeyMap = ofy().load().keys(() -> testCaseKeyList.iterator()); + for (Map.Entry<com.googlecode.objectify.Key<TestCaseRunEntity>, TestCaseRunEntity> entry : + testCaseRunEntityKeyMap.entrySet()) { + details.addTestCase(entry.getValue()); } - details.addTestCase(testCaseRun); } return details; } diff --git a/src/main/java/com/android/vts/api/UserFavoriteRestServlet.java b/src/main/java/com/android/vts/api/UserFavoriteRestServlet.java index bbb95da..30fd649 100644 --- a/src/main/java/com/android/vts/api/UserFavoriteRestServlet.java +++ b/src/main/java/com/android/vts/api/UserFavoriteRestServlet.java @@ -45,7 +45,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; /** Servlet for handling requests to add or remove subscriptions. */ -public class UserFavoriteRestServlet extends HttpServlet { +public class UserFavoriteRestServlet extends BaseApiServlet { protected static final Logger logger = Logger.getLogger(UserFavoriteRestServlet.class.getName()); diff --git a/src/main/java/com/android/vts/job/VtsInactivityJobServlet.java b/src/main/java/com/android/vts/job/VtsInactivityJobServlet.java index ce2b77a..c4a9825 100644 --- a/src/main/java/com/android/vts/job/VtsInactivityJobServlet.java +++ b/src/main/java/com/android/vts/job/VtsInactivityJobServlet.java @@ -77,7 +77,7 @@ public class VtsInactivityJobServlet extends HttpServlet { // After 7 full days have passed, notifications will no longer be sent (i.e. the // test is assumed to be deprecated). if (diff >= TimeUnit.DAYS.toMicros(1) && diff < TimeUnit.DAYS.toMicros(8)) { - String uploadTimeString = TimeUtil.getDateTimeString(lastRunTime); + String uploadTimeString = TimeUtil.getDateTimeZoneString(lastRunTime); String subject = "Warning! Inactive test: " + test.getTestName(); String body = "Hello,<br><br>Test \"" diff --git a/src/main/java/com/android/vts/servlet/ShowCoverageOverviewServlet.java b/src/main/java/com/android/vts/servlet/ShowCoverageOverviewServlet.java index f0cafbe..719ade9 100644 --- a/src/main/java/com/android/vts/servlet/ShowCoverageOverviewServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowCoverageOverviewServlet.java @@ -16,26 +16,16 @@ package com.android.vts.servlet; +import com.android.vts.entity.CodeCoverageEntity; import com.android.vts.entity.TestCoverageStatusEntity; import com.android.vts.entity.TestEntity; import com.android.vts.entity.TestRunEntity; -import com.android.vts.entity.TestRunEntity.TestRunType; + import com.android.vts.proto.VtsReportMessage; -import com.android.vts.util.DatastoreHelper; import com.android.vts.util.FilterUtil; -import com.android.vts.util.TestRunMetadata; -import com.google.appengine.api.datastore.DatastoreService; -import com.google.appengine.api.datastore.DatastoreServiceFactory; -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.Key; -import com.google.appengine.api.datastore.KeyFactory; -import com.google.appengine.api.datastore.Query; import com.google.cloud.datastore.Datastore; import com.google.cloud.datastore.DatastoreOptions; -import com.google.cloud.datastore.LongValue; import com.google.cloud.datastore.PathElement; -import com.google.cloud.datastore.QueryResults; -import com.google.cloud.datastore.StringValue; import com.google.cloud.datastore.StructuredQuery.CompositeFilter; import com.google.cloud.datastore.StructuredQuery.Filter; import com.google.cloud.datastore.StructuredQuery.PropertyFilter; @@ -52,30 +42,33 @@ import com.google.visualization.datasource.datatable.ColumnDescription; import com.google.visualization.datasource.datatable.DataTable; import com.google.visualization.datasource.datatable.TableRow; import com.google.visualization.datasource.datatable.value.DateTimeValue; -import com.google.visualization.datasource.datatable.value.DateValue; import com.google.visualization.datasource.datatable.value.NumberValue; import com.google.visualization.datasource.datatable.value.ValueType; +import com.googlecode.objectify.Key; import com.ibm.icu.util.GregorianCalendar; import com.ibm.icu.util.TimeZone; import java.io.IOException; import java.math.BigDecimal; import java.math.RoundingMode; +import java.time.Duration; +import java.time.Instant; import java.util.ArrayList; import java.util.Calendar; +import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.servlet.RequestDispatcher; import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.joda.time.DateTime; + import org.joda.time.format.DateTimeFormat; import org.joda.time.format.DateTimeFormatter; +import static com.googlecode.objectify.ObjectifyService.ofy; + /** * Represents the servlet that is invoked on loading the coverage overview page. */ @@ -148,11 +141,26 @@ public class ShowCoverageOverviewServlet extends BaseServlet { } } + private List<Key<TestRunEntity>> getTestCoverageStatusEntityKeyList( + List<TestCoverageStatusEntity> testCoverageStatusEntityList) { + return testCoverageStatusEntityList.stream() + .map( + testCoverageStatusEntity -> { + com.googlecode.objectify.Key testKey = + com.googlecode.objectify.Key.create( + TestEntity.class, + testCoverageStatusEntity.getTestName()); + return com.googlecode.objectify.Key.create( + testKey, + TestRunEntity.class, + testCoverageStatusEntity.getUpdatedTimestamp()); + }) + .collect(Collectors.toList()); + } + private RequestDispatcher getCoverageDispatcher( HttpServletRequest request, HttpServletResponse response) { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - String COVERAGE_OVERVIEW_JSP = "WEB-INF/jsp/show_coverage_overview.jsp"; RequestDispatcher dispatcher = null; @@ -171,70 +179,58 @@ public class ShowCoverageOverviewServlet extends BaseServlet { showPresubmit = true; } - Map<String, TestCoverageStatusEntity> testCoverageStatusMap = TestCoverageStatusEntity - .getTestCoverageStatusMap(); - - List<Key> allTests = TestEntity.getAllTest().stream().map(t -> t.getOldKey()) - .collect(Collectors.toList()); - // Add test names to list List<String> resultNames = new ArrayList<>(); for (VtsReportMessage.TestCaseResult r : VtsReportMessage.TestCaseResult.values()) { resultNames.add(r.name()); } - List<JsonObject> testRunObjects = new ArrayList<>(); + Map<String, String[]> parameterMap = request.getParameterMap(); - Query.Filter testFilter = - new Query.FilterPredicate( - TestRunEntity.HAS_COVERAGE, Query.FilterOperator.EQUAL, true); - Query.Filter timeFilter = - FilterUtil.getTestTypeFilter(showPresubmit, showPostsubmit, unfiltered); + List<TestCoverageStatusEntity> testCoverageStatusEntityList = + TestCoverageStatusEntity.getAllTestCoverage(); + + List<com.googlecode.objectify.Key<TestRunEntity>> testCoverageStatusEntityKeyList = + this.getTestCoverageStatusEntityKeyList(testCoverageStatusEntityList); + + Map<Key<TestRunEntity>, TestRunEntity> keyTestRunEntityMap = + ofy().load().keys(() -> testCoverageStatusEntityKeyList.iterator()); + + List<com.googlecode.objectify.Key<CodeCoverageEntity>> codeCoverageEntityKeyList = + new ArrayList<>(); + Map<Long, TestRunEntity> testRunEntityMap = new HashMap<>(); + for (Map.Entry<com.googlecode.objectify.Key<TestRunEntity>, TestRunEntity> entry : + keyTestRunEntityMap.entrySet()) { + com.googlecode.objectify.Key codeCoverageEntityKey = + com.googlecode.objectify.Key.create( + entry.getKey(), CodeCoverageEntity.class, entry.getValue().getId()); + codeCoverageEntityKeyList.add(codeCoverageEntityKey); + testRunEntityMap.put(entry.getValue().getId(), entry.getValue()); + } - if (timeFilter != null) { - testFilter = Query.CompositeFilterOperator.and(testFilter, timeFilter); - } - Map<String, String[]> parameterMap = request.getParameterMap(); - List<Query.Filter> userTestFilters = FilterUtil.getUserTestFilters(parameterMap); - userTestFilters.add(0, testFilter); - Query.Filter userDeviceFilter = FilterUtil.getUserDeviceFilter(parameterMap); + Map<com.googlecode.objectify.Key<CodeCoverageEntity>, CodeCoverageEntity> + keyCodeCoverageEntityMap = + ofy().load().keys(() -> codeCoverageEntityKeyList.iterator()); + + Map<Long, CodeCoverageEntity> codeCoverageEntityMap = new HashMap<>(); + for (Map.Entry<com.googlecode.objectify.Key<CodeCoverageEntity>, CodeCoverageEntity> entry : + keyCodeCoverageEntityMap.entrySet()) { + codeCoverageEntityMap.put(entry.getValue().getId(), entry.getValue()); + } int coveredLines = 0; int uncoveredLines = 0; int passCount = 0; int failCount = 0; - for (Key key : allTests) { - List<Key> gets = - FilterUtil.getMatchingKeys( - key, - TestRunEntity.KIND, - userTestFilters, - userDeviceFilter, - Query.SortDirection.DESCENDING, - 1); - Map<Key, Entity> entityMap = datastore.get(gets); - for (Key entityKey : gets) { - if (!entityMap.containsKey(entityKey)) { - continue; - } - TestRunEntity testRunEntity = TestRunEntity.fromEntity(entityMap.get(entityKey)); - if (testRunEntity == null) { - continue; - } + for (Map.Entry<Long, CodeCoverageEntity> entry : codeCoverageEntityMap.entrySet()) { + TestRunEntity testRunEntity = testRunEntityMap.get(entry.getKey()); - // Overwrite the coverage value with newly update value from user decision - TestCoverageStatusEntity testCoverageStatusEntity = testCoverageStatusMap - .get(key.getName()); - testRunEntity.setCoveredLineCount(testCoverageStatusEntity.getUpdatedCoveredLineCount()); - testRunEntity.setTotalLineCount(testCoverageStatusEntity.getUpdatedTotalLineCount()); - TestRunMetadata metadata = new TestRunMetadata(key.getName(), testRunEntity); + CodeCoverageEntity codeCoverageEntity = entry.getValue(); - testRunObjects.add(metadata.toJson()); - coveredLines += testRunEntity.getCoveredLineCount(); - uncoveredLines += testRunEntity.getTotalLineCount() - testRunEntity.getCoveredLineCount(); + coveredLines += codeCoverageEntity.getCoveredLineCount(); + uncoveredLines += codeCoverageEntity.getTotalLineCount() - codeCoverageEntity.getCoveredLineCount(); passCount += testRunEntity.getPassCount(); failCount += testRunEntity.getFailCount(); - } } FilterUtil.setAttributes(request, parameterMap); @@ -246,7 +242,8 @@ public class ShowCoverageOverviewServlet extends BaseServlet { response.setStatus(HttpServletResponse.SC_OK); request.setAttribute("resultNames", resultNames); request.setAttribute("resultNamesJson", new Gson().toJson(resultNames)); - request.setAttribute("testRuns", new Gson().toJson(testRunObjects)); + request.setAttribute("testRunEntityList", testRunEntityMap.values()); + request.setAttribute("codeCoverageEntityMap", codeCoverageEntityMap); request.setAttribute("coveredLines", new Gson().toJson(coveredLines)); request.setAttribute("uncoveredLines", new Gson().toJson(uncoveredLines)); request.setAttribute("testStats", new Gson().toJson(testStats)); @@ -254,8 +251,8 @@ public class ShowCoverageOverviewServlet extends BaseServlet { request.setAttribute("unfiltered", unfiltered); request.setAttribute("showPresubmit", showPresubmit); request.setAttribute("showPostsubmit", showPostsubmit); - request.setAttribute("branches", new Gson().toJson(DatastoreHelper.getAllBranches())); - request.setAttribute("devices", new Gson().toJson(DatastoreHelper.getAllBuildFlavors())); + request.setAttribute("branches", new Gson().toJson(new ArrayList<>())); // DeviceInfoEntity.getAllBranches() + request.setAttribute("devices", new Gson().toJson(new ArrayList<>())); // DeviceInfoEntity.getAllBuildFlavors() dispatcher = request.getRequestDispatcher(COVERAGE_OVERVIEW_JSP); return dispatcher; } @@ -278,7 +275,7 @@ public class ShowCoverageOverviewServlet extends BaseServlet { dataTable.addColumns(cd); Calendar cal = Calendar.getInstance(); - cal.add(Calendar.DATE, -30); + cal.add(Calendar.MONTH, -6); Long startTime = cal.getTime().getTime() * 1000; Long endTime = Calendar.getInstance().getTime().getTime() * 1000; @@ -294,79 +291,55 @@ public class ShowCoverageOverviewServlet extends BaseServlet { PathElement.of(TestRunEntity.KIND, endTime)) .newKey(endTime); - Filter testRunFilter = CompositeFilter.and( + Filter codeCoverageFilter = CompositeFilter.and( PropertyFilter.lt("__key__", endKey), - PropertyFilter.gt("__key__", startKey), - PropertyFilter.eq("hasCoverage", true) + PropertyFilter.gt("__key__", startKey) ); - com.google.cloud.datastore.Query<com.google.cloud.datastore.Entity> testRunQuery = - com.google.cloud.datastore.Query - .newEntityQueryBuilder() - .setKind(TestRunEntity.KIND) - .setFilter(testRunFilter) - .build(); - - List<TestRunEntity> testRunEntityList = new ArrayList<>(); - QueryResults<com.google.cloud.datastore.Entity> testRunIterator = datastore.run(testRunQuery); - while (testRunIterator.hasNext()) { - com.google.cloud.datastore.Entity entity = testRunIterator.next(); - - Key parentKey = KeyFactory.createKey(TestEntity.KIND, entity.getString("testName")); - - List<LongValue> testCaseIdList = entity.getList("testCaseIds"); - List<StringValue> linkList = new ArrayList<>(); - if (entity.contains("logLinks")) { - linkList = entity.getList("logLinks"); - } - TestRunEntity testRunEntity = new TestRunEntity( - KeyFactory.createKey(parentKey, TestRunEntity.KIND, entity.getLong("startTimestamp")), - TestRunType.fromNumber(Long.valueOf(entity.getLong("type")).intValue()), - entity.getLong("startTimestamp"), - entity.getLong("endTimestamp"), - entity.getString("testBuildId"), - entity.getString("hostName"), - entity.getLong("passCount"), - entity.getLong("failCount"), - testCaseIdList.stream().map(value -> value.get()).collect(Collectors.toList()), - linkList.stream().map(v -> v.get()).collect(Collectors.toList()), - entity.getLong("coveredLineCount"), - entity.getLong("totalLineCount") - ); - testRunEntityList.add(testRunEntity); - } + List<CodeCoverageEntity> codeCoverageEntityList = ofy().load() + .type(CodeCoverageEntity.class) + .filter(codeCoverageFilter) + .limit(10) + .list(); DateTimeFormatter dateTimeFormatter = DateTimeFormat.forPattern("yyyy-MM-dd"); - Map<String, List<TestRunEntity>> testRunEntityListMap = testRunEntityList.stream().collect( - Collectors.groupingBy(v -> dateTimeFormatter.print(v.getStartTimestamp() / 1000)) + Map<String, List<CodeCoverageEntity>> codeCoverageEntityListMap = codeCoverageEntityList.stream().collect( + Collectors.groupingBy(v -> dateTimeFormatter.print(v.getId() / 1000)) ); - testRunEntityListMap.forEach((key, entityList) -> { - if (dataTable.getRows().size() < 10) { - GregorianCalendar gCal = new GregorianCalendar(); - gCal.setTimeZone(TimeZone.getTimeZone("GMT")); - gCal.setTimeInMillis(entityList.get(0).getStartTimestamp() / 1000); - - Long sumCoveredLine = entityList.stream().mapToLong(val -> val.getCoveredLineCount()).sum(); - Long sumTotalLine = entityList.stream().mapToLong(val -> val.getTotalLineCount()).sum(); - BigDecimal coveredLineNum = new BigDecimal(sumCoveredLine); - BigDecimal totalLineNum = new BigDecimal(sumTotalLine); - BigDecimal totalPercent = new BigDecimal(100); - float percentage = coveredLineNum.multiply(totalPercent).divide(totalLineNum, 2, - RoundingMode.HALF_DOWN).floatValue(); - - TableRow tableRow = new TableRow(); - tableRow.addCell(new DateTimeValue(gCal)); - tableRow.addCell(new NumberValue(sumCoveredLine)); - tableRow.addCell(new NumberValue(sumTotalLine)); - tableRow.addCell(new NumberValue(percentage)); - try { - dataTable.addRow(tableRow); - } catch (TypeMismatchException e) { - logger.log(Level.WARNING, "Invalid type! "); - } - } - }); + codeCoverageEntityListMap.forEach( + (key, entityList) -> { + GregorianCalendar gCal = new GregorianCalendar(); + gCal.setTimeZone(TimeZone.getTimeZone("GMT")); + gCal.setTimeInMillis(entityList.get(0).getId() / 1000); + + Long sumCoveredLine = + entityList.stream().mapToLong(val -> val.getCoveredLineCount()).sum(); + Long sumTotalLine = + entityList.stream().mapToLong(val -> val.getTotalLineCount()).sum(); + float percentage = 0; + if (sumTotalLine > 0) { + BigDecimal coveredLineNum = new BigDecimal(sumCoveredLine); + BigDecimal totalLineNum = new BigDecimal(sumTotalLine); + BigDecimal totalPercent = new BigDecimal(100); + percentage = + coveredLineNum + .multiply(totalPercent) + .divide(totalLineNum, 2, RoundingMode.HALF_DOWN) + .floatValue(); + } + + TableRow tableRow = new TableRow(); + tableRow.addCell(new DateTimeValue(gCal)); + tableRow.addCell(new NumberValue(sumCoveredLine)); + tableRow.addCell(new NumberValue(sumTotalLine)); + tableRow.addCell(new NumberValue(percentage)); + try { + dataTable.addRow(tableRow); + } catch (TypeMismatchException e) { + logger.log(Level.WARNING, "Invalid type! "); + } + }); return dataTable; } diff --git a/src/main/java/com/android/vts/servlet/ShowPlanReleaseServlet.java b/src/main/java/com/android/vts/servlet/ShowPlanReleaseServlet.java index acccf8b..0fa28f2 100644 --- a/src/main/java/com/android/vts/servlet/ShowPlanReleaseServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowPlanReleaseServlet.java @@ -17,14 +17,19 @@ package com.android.vts.servlet; import com.android.vts.entity.DeviceInfoEntity; -import com.android.vts.entity.ProfilingPointSummaryEntity; import com.android.vts.entity.TestPlanEntity; import com.android.vts.entity.TestPlanRunEntity; import com.android.vts.entity.TestSuiteResultEntity; import com.android.vts.util.DatastoreHelper; import com.android.vts.util.FilterUtil; import com.android.vts.util.Pagination; -import com.google.appengine.api.datastore.*; + +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Entity; +import com.google.appengine.api.datastore.Key; +import com.google.appengine.api.datastore.KeyFactory; +import com.google.appengine.api.datastore.Query; import com.google.appengine.api.datastore.Query.Filter; import com.google.appengine.api.datastore.Query.SortDirection; import com.google.gson.Gson; @@ -37,7 +42,16 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.IOException; -import java.util.*; + +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.logging.Level; import java.util.stream.Collectors; import java.util.stream.IntStream; diff --git a/src/main/java/com/android/vts/servlet/ShowTableServlet.java b/src/main/java/com/android/vts/servlet/ShowTableServlet.java index cfe85c3..b56ce9c 100644 --- a/src/main/java/com/android/vts/servlet/ShowTableServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowTableServlet.java @@ -16,6 +16,7 @@ package com.android.vts.servlet; +import com.android.vts.entity.CodeCoverageEntity; import com.android.vts.entity.TestCaseRunEntity; import com.android.vts.entity.TestEntity; import com.android.vts.entity.TestRunEntity; diff --git a/src/main/java/com/android/vts/servlet/ShowTreeServlet.java b/src/main/java/com/android/vts/servlet/ShowTreeServlet.java index d0a682e..e087807 100644 --- a/src/main/java/com/android/vts/servlet/ShowTreeServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowTreeServlet.java @@ -85,9 +85,9 @@ public class ShowTreeServlet extends BaseServlet { DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); TestRunDetails details = new TestRunDetails(); List<Key> gets = new ArrayList<>(); - for (long testCaseId : metadata.testRun.getTestCaseIds()) { - gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); - } + for (long testCaseId : metadata.testRun.getTestCaseIds()) { + gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); + } Map<Key, Entity> entityMap = datastore.get(gets); for (int i = 0; i < 1; i++) { for (Key key : entityMap.keySet()) { diff --git a/src/main/java/com/android/vts/util/DatastoreHelper.java b/src/main/java/com/android/vts/util/DatastoreHelper.java index cb511ab..4ba19d0 100644 --- a/src/main/java/com/android/vts/util/DatastoreHelper.java +++ b/src/main/java/com/android/vts/util/DatastoreHelper.java @@ -13,9 +13,11 @@ */ package com.android.vts.util; + import com.android.vts.entity.ApiCoverageEntity; import com.android.vts.entity.BranchEntity; import com.android.vts.entity.BuildTargetEntity; +import com.android.vts.entity.CodeCoverageEntity; import com.android.vts.entity.CoverageEntity; import com.android.vts.entity.DeviceInfoEntity; import com.android.vts.entity.ProfilingPointRunEntity; @@ -223,6 +225,7 @@ public class DatastoreHelper { String systraceLink = testCase.getSystraceList().get(0).getUrl(0).toStringUtf8(); links.add(systraceLink); } + // Process coverage data for test case for (CoverageReportMessage coverage : testCase.getCoverageList()) { CoverageEntity coverageEntity = @@ -270,7 +273,7 @@ public class DatastoreHelper { } // Process device information - TestRunType testRunType = null; + long testRunType = 0; for (AndroidDeviceInfoMessage device : report.getDeviceInfoList()) { DeviceInfoEntity deviceInfoEntity = DeviceInfoEntity.fromDeviceInfoMessage(testRunKey, device); @@ -279,10 +282,10 @@ public class DatastoreHelper { } else { // Run type on devices must be the same, else set to OTHER TestRunType runType = TestRunType.fromBuildId(deviceInfoEntity.getBuildId()); - if (testRunType == null) { - testRunType = runType; - } else if (runType != testRunType) { - testRunType = TestRunType.OTHER; + if (runType == null) { + testRunType = TestRunType.OTHER.getNumber(); + } else { + testRunType = runType.getNumber(); } testEntityList.add(deviceInfoEntity.toEntity()); BuildTargetEntity target = new BuildTargetEntity(deviceInfoEntity.getBuildFlavor()); @@ -297,10 +300,10 @@ public class DatastoreHelper { } // Overall run type should be determined by the device builds unless test build is OTHER - if (testRunType == null) { - testRunType = TestRunType.fromBuildId(testBuildId); + if (testRunType == TestRunType.OTHER.getNumber()) { + testRunType = TestRunType.fromBuildId(testBuildId).getNumber(); } else if (TestRunType.fromBuildId(testBuildId) == TestRunType.OTHER) { - testRunType = TestRunType.OTHER; + testRunType = TestRunType.OTHER.getNumber(); } // Process global coverage data @@ -366,6 +369,7 @@ public class DatastoreHelper { } } + boolean hasCodeCoverage = totalLineCount > 0 && coveredLineCount >= 0; TestRunEntity testRunEntity = new TestRunEntity( testEntity.getOldKey(), @@ -376,11 +380,16 @@ public class DatastoreHelper { hostName, passCount, failCount, + hasCodeCoverage, testCaseIds, - links, + links); + testEntityList.add(testRunEntity.toEntity()); + + CodeCoverageEntity codeCoverageEntity = new CodeCoverageEntity( + testRunEntity.getKey(), coveredLineCount, totalLineCount); - testEntityList.add(testRunEntity.toEntity()); + testEntityList.add(codeCoverageEntity.toEntity()); Entity test = testEntity.toEntity(); @@ -419,9 +428,9 @@ public class DatastoreHelper { indexCount++; } - if (testRunEntity.getType() == TestRunType.POSTSUBMIT) { + if (testRunEntity.getType() == TestRunType.POSTSUBMIT.getNumber()) { VtsAlertJobServlet.addTask(testRunKey); - if (testRunEntity.isHasCoverage()) { + if (testRunEntity.isHasCodeCoverage()) { VtsCoverageAlertJobServlet.addTask(testRunKey); } if (profilingPointKeys.size() > 0) { @@ -467,7 +476,7 @@ public class DatastoreHelper { long startTimestamp = -1; long endTimestamp = -1; String testBuildId = null; - TestRunType type = null; + long type = 0; Set<DeviceInfoEntity> deviceInfoEntitySet = new HashSet<>(); for (Key testRunKey : testRuns.keySet()) { TestRunEntity testRun = TestRunEntity.fromEntity(testRuns.get(testRunKey)); @@ -482,11 +491,7 @@ public class DatastoreHelper { if (endTimestamp < 0 || testRun.getEndTimestamp() > endTimestamp) { endTimestamp = testRun.getEndTimestamp(); } - if (type == null) { - type = testRun.getType(); - } else if (type != testRun.getType()) { - type = TestRunType.OTHER; - } + type = testRun.getType(); testBuildId = testRun.getTestBuildId(); Query deviceInfoQuery = new Query(DeviceInfoEntity.KIND).setAncestor(testRunKey); for (Entity deviceInfoEntity : datastore.prepare(deviceInfoQuery).asIterable()) { @@ -497,7 +502,7 @@ public class DatastoreHelper { deviceInfoEntitySet.add(device); } } - if (startTimestamp < 0 || testBuildId == null || type == null) { + if (startTimestamp < 0 || testBuildId == null || type == 0) { logger.log(Level.WARNING, "Couldn't infer test run information from runs."); return; } diff --git a/src/main/java/com/android/vts/util/EmailHelper.java b/src/main/java/com/android/vts/util/EmailHelper.java index 0bec72b..de1d6e9 100644 --- a/src/main/java/com/android/vts/util/EmailHelper.java +++ b/src/main/java/com/android/vts/util/EmailHelper.java @@ -26,9 +26,11 @@ import com.google.appengine.api.datastore.Query.FilterOperator; import com.google.appengine.api.datastore.Query.FilterPredicate; import java.io.IOException; import java.io.UnsupportedEncodingException; + import java.util.ArrayList; import java.util.HashSet; import java.util.List; +import java.util.Optional; import java.util.Properties; import java.util.Set; import java.util.logging.Level; @@ -75,8 +77,8 @@ public class EmailHelper { if (testRun != null) { sb.append("VTS Build ID: " + testRun.getTestBuildId() + "<br>"); - sb.append("Start Time: " + TimeUtil.getDateTimeString(testRun.getStartTimestamp())); - sb.append("<br>End Time: " + TimeUtil.getDateTimeString(testRun.getEndTimestamp())); + sb.append("Start Time: " + TimeUtil.getDateTimeZoneString(testRun.getStartTimestamp())); + sb.append("<br>End Time: " + TimeUtil.getDateTimeZoneString(testRun.getEndTimestamp())); } sb.append( "<br><br>For details, visit the" @@ -105,11 +107,15 @@ public class EmailHelper { } for (Entity favorite : datastore.prepare(favoritesQuery).asIterable()) { UserFavoriteEntity favoriteEntity = UserFavoriteEntity.fromEntity(favorite); + // TODO this logic need to be reexamined thoroughly and improved if (favoriteEntity != null && favoriteEntity.user != null - && favoriteEntity.user.getEmail().endsWith(EMAIL_DOMAIN) && !favoriteEntity.muteNotifications) { - emailSet.add(favoriteEntity.user.getEmail()); + Optional<String> userEmail = Optional.of(favoriteEntity.user.getEmail()); + if (userEmail.isPresent() && + userEmail.orElse("").endsWith(EMAIL_DOMAIN)) { + emailSet.add(favoriteEntity.user.getEmail()); + } } } return new ArrayList<>(emailSet); diff --git a/src/main/java/com/android/vts/util/TestResults.java b/src/main/java/com/android/vts/util/TestResults.java index b6cfc2c..38617c3 100644 --- a/src/main/java/com/android/vts/util/TestResults.java +++ b/src/main/java/com/android/vts/util/TestResults.java @@ -16,6 +16,7 @@ package com.android.vts.util; +import com.android.vts.entity.CodeCoverageEntity; import com.android.vts.entity.DeviceInfoEntity; import com.android.vts.entity.ProfilingPointRunEntity; import com.android.vts.entity.TestCaseRunEntity; @@ -251,6 +252,7 @@ public class TestResults { // Iterate through the test runs for (int col = 0; col < testRuns.size(); col++) { TestRunEntity testRun = testRuns.get(col); + CodeCoverageEntity codeCoverageEntity = testRun.getCodeCoverageEntity(); // Process the device information List<DeviceInfoEntity> devices = deviceInfoMap.get(testRun.getKey()); @@ -287,8 +289,13 @@ public class TestResults { int passCount = (int) testRun.getPassCount(); int nonpassCount = (int) testRun.getFailCount(); TestCaseResult aggregateStatus = TestCaseResult.UNKNOWN_RESULT; - long totalLineCount = testRun.getTotalLineCount(); - long coveredLineCount = testRun.getCoveredLineCount(); + + long totalLineCount = 0; + long coveredLineCount = 0; + if (testRun.isHasCodeCoverage()) { + totalLineCount = codeCoverageEntity.getTotalLineCount(); + coveredLineCount = codeCoverageEntity.getCoveredLineCount(); + } // Process test case results for (TestCaseRunEntity testCaseEntity : testCaseRunMap.get(testRun.getKey())) { @@ -372,8 +379,8 @@ public class TestResults { List<String[]> linkEntries = new ArrayList<>(); logInfoMap.put(Integer.toString(col), linkEntries); - if (testRun.getLinks() != null) { - for (String rawUrl : testRun.getLinks()) { + if (testRun.getLogLinks() != null) { + for (String rawUrl : testRun.getLogLinks()) { LinkDisplay validatedLink = UrlUtil.processUrl(rawUrl); if (validatedLink == null) { logger.log(Level.WARNING, "Invalid logging URL : " + rawUrl); diff --git a/src/main/java/com/android/vts/util/TestRunDetails.java b/src/main/java/com/android/vts/util/TestRunDetails.java index 60b2d90..584abaf 100644 --- a/src/main/java/com/android/vts/util/TestRunDetails.java +++ b/src/main/java/com/android/vts/util/TestRunDetails.java @@ -73,7 +73,7 @@ public class TestRunDetails { * @param testCaseEntity The TestCaseRunEntity object storing test case results. */ public void addTestCase(TestCaseRunEntity testCaseEntity) { - for (TestCase testCase : testCaseEntity.testCases) { + for (TestCase testCase : testCaseEntity.getTestCases()) { int result = testCase.result; if (result > resultCounts.length) continue; diff --git a/src/main/java/com/android/vts/util/TimeUtil.java b/src/main/java/com/android/vts/util/TimeUtil.java index 46ed55a..7e9be26 100644 --- a/src/main/java/com/android/vts/util/TimeUtil.java +++ b/src/main/java/com/android/vts/util/TimeUtil.java @@ -27,19 +27,20 @@ public class TimeUtil { protected static final Logger logger = Logger.getLogger(TimeUtil.class.getName()); public static final String DATE_FORMAT = "yyyy-MM-dd"; - public static final String DATE_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss (z)"; + public static final String DATE_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; + public static final String DATE_TIME_ZONE_FORMAT = "yyyy-MM-dd HH:mm:ss (z)"; public static final ZoneId PT_ZONE = ZoneId.of("America/Los_Angeles"); /** - * Create a date time string from the provided timestamp. + * Create a date time string with zone info from the provided timestamp. * * @param timeMicroseconds The time in microseconds * @return A formatted date time string. */ - public static String getDateTimeString(long timeMicroseconds) { + public static String getDateTimeZoneString(long timeMicroseconds) { long timeMillis = TimeUnit.MICROSECONDS.toMillis(timeMicroseconds); ZonedDateTime zdt = ZonedDateTime.ofInstant(Instant.ofEpochMilli(timeMillis), PT_ZONE); - return DateTimeFormatter.ofPattern(DATE_TIME_FORMAT).format(zdt); + return DateTimeFormatter.ofPattern(DATE_TIME_ZONE_FORMAT).format(zdt); } /** @@ -55,6 +56,18 @@ public class TimeUtil { } /** + * Create a date string from the provided timestamp. + * + * @param timeMicroseconds The time in microseconds + * @return A formatted date string. + */ + public static String getDateTimeString(long timeMicroseconds) { + long timeMillis = TimeUnit.MICROSECONDS.toMillis(timeMicroseconds); + ZonedDateTime zdt = ZonedDateTime.ofInstant(Instant.ofEpochMilli(timeMillis), PT_ZONE); + return DateTimeFormatter.ofPattern(DATE_TIME_FORMAT).format(zdt); + } + + /** * Create a ZonedDateTime object from the provided timestamp. * * @param timeMicroseconds The time in microseconds |