diff options
Diffstat (limited to 'src/main/java/com/android/vts/job')
8 files changed, 0 insertions, 2127 deletions
diff --git a/src/main/java/com/android/vts/job/BaseJobServlet.java b/src/main/java/com/android/vts/job/BaseJobServlet.java deleted file mode 100644 index 18a1d24..0000000 --- a/src/main/java/com/android/vts/job/BaseJobServlet.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2018 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may - * obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.android.vts.job; - - -import javax.servlet.ServletConfig; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServlet; -import java.util.Properties; - -/** - * An abstract class to be subclassed to create Job Servlet - */ -public abstract class BaseJobServlet extends HttpServlet { - - /** - * System Configuration Property class - */ - protected static Properties systemConfigProp = new Properties(); - - /** - * This variable is for maximum number of entities per transaction You can find the detail here - * (https://cloud.google.com/datastore/docs/concepts/limits) - */ - protected int MAX_ENTITY_SIZE_PER_TRANSACTION = 300; - - @Override - public void init(ServletConfig cfg) throws ServletException { - super.init(cfg); - - systemConfigProp = - Properties.class.cast(cfg.getServletContext().getAttribute("systemConfigProp")); - - this.MAX_ENTITY_SIZE_PER_TRANSACTION = - Integer.parseInt(systemConfigProp.getProperty("datastore.maxEntitySize")); - } -} diff --git a/src/main/java/com/android/vts/job/VtsAlertJobServlet.java b/src/main/java/com/android/vts/job/VtsAlertJobServlet.java deleted file mode 100644 index be23982..0000000 --- a/src/main/java/com/android/vts/job/VtsAlertJobServlet.java +++ /dev/null @@ -1,582 +0,0 @@ -/* - * Copyright (c) 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may - * obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.android.vts.job; - -import com.android.vts.entity.DeviceInfoEntity; -import com.android.vts.entity.TestAcknowledgmentEntity; -import com.android.vts.entity.TestCaseRunEntity; -import com.android.vts.entity.TestCaseRunEntity.TestCase; -import com.android.vts.entity.TestRunEntity; -import com.android.vts.entity.TestStatusEntity; -import com.android.vts.entity.TestStatusEntity.TestCaseReference; -import com.android.vts.proto.VtsReportMessage.TestCaseResult; -import com.android.vts.util.DatastoreHelper; -import com.android.vts.util.EmailHelper; -import com.android.vts.util.FilterUtil; -import com.android.vts.util.TimeUtil; -import com.google.appengine.api.datastore.DatastoreFailureException; -import com.google.appengine.api.datastore.DatastoreService; -import com.google.appengine.api.datastore.DatastoreServiceFactory; -import com.google.appengine.api.datastore.DatastoreTimeoutException; -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityNotFoundException; -import com.google.appengine.api.datastore.FetchOptions; -import com.google.appengine.api.datastore.Key; -import com.google.appengine.api.datastore.KeyFactory; -import com.google.appengine.api.datastore.Query; -import com.google.appengine.api.datastore.Query.Filter; -import com.google.appengine.api.datastore.Query.SortDirection; -import com.google.appengine.api.datastore.Transaction; -import com.google.appengine.api.taskqueue.Queue; -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TaskOptions; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Comparator; -import java.util.ConcurrentModificationException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.mail.Message; -import javax.mail.MessagingException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.apache.commons.lang.StringUtils; - -/** Represents the notifications service which is automatically called on a fixed schedule. */ -public class VtsAlertJobServlet extends BaseJobServlet { - private static final String ALERT_JOB_URL = "/task/vts_alert_job"; - protected static final Logger logger = Logger.getLogger(VtsAlertJobServlet.class.getName()); - protected static final int MAX_RUN_COUNT = 1000; // maximum number of runs to query for - - /** - * Process the current test case failures for a test. - * - * @param status The TestStatusEntity object for the test. - * @returns a map from test case name to the test case run ID for which the test case failed. - */ - private static Map<String, TestCase> getCurrentFailures(TestStatusEntity status) { - if (status.getFailingTestCases() == null || status.getFailingTestCases().size() == 0) { - return new HashMap<>(); - } - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - Map<String, TestCase> failingTestcases = new HashMap<>(); - Set<Key> gets = new HashSet<>(); - for (TestCaseReference testCaseRef : status.getFailingTestCases()) { - gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId)); - } - if (gets.size() == 0) { - return failingTestcases; - } - Map<Key, Entity> testCaseMap = datastore.get(gets); - - for (TestCaseReference testCaseRef : status.getFailingTestCases()) { - Key key = KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId); - if (!testCaseMap.containsKey(key)) { - continue; - } - Entity testCaseRun = testCaseMap.get(key); - TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun); - if (testCaseRunEntity.testCases.size() <= testCaseRef.offset) { - continue; - } - TestCase testCase = testCaseRunEntity.testCases.get(testCaseRef.offset); - failingTestcases.put(testCase.name, testCase); - } - return failingTestcases; - } - - /** - * Get the test acknowledgments for a test key. - * - * @param testKey The key to the test whose acknowledgments to fetch. - * @return A list of test acknowledgments. - */ - private static List<TestAcknowledgmentEntity> getTestCaseAcknowledgments(Key testKey) { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - - List<TestAcknowledgmentEntity> acks = new ArrayList<>(); - Filter testFilter = - new Query.FilterPredicate( - TestAcknowledgmentEntity.TEST_KEY, Query.FilterOperator.EQUAL, testKey); - Query q = new Query(TestAcknowledgmentEntity.KIND).setFilter(testFilter); - - for (Entity ackEntity : datastore.prepare(q).asIterable()) { - TestAcknowledgmentEntity ack = TestAcknowledgmentEntity.fromEntity(ackEntity); - if (ack == null) continue; - acks.add(ack); - } - return acks; - } - - /** - * Get the test runs for the test in the specified time window. - * - * <p>If the start and end time delta is greater than one day, the query will be truncated. - * - * @param testKey The key to the test whose runs to query. - * @param startTime The start time for the query. - * @param endTime The end time for the query. - * @return A list of test runs in the specified time window. - */ - private static List<TestRunEntity> getTestRuns(Key testKey, long startTime, long endTime) { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - Filter testTypeFilter = FilterUtil.getTestTypeFilter(false, true, false); - long delta = endTime - startTime; - delta = Math.min(delta, TimeUnit.DAYS.toMicros(1)); - Filter runFilter = - FilterUtil.getTimeFilter( - testKey, TestRunEntity.KIND, endTime - delta + 1, endTime, testTypeFilter); - - Query q = - new Query(TestRunEntity.KIND) - .setAncestor(testKey) - .setFilter(runFilter) - .addSort(Entity.KEY_RESERVED_PROPERTY, SortDirection.DESCENDING); - - List<TestRunEntity> testRuns = new ArrayList<>(); - for (Entity testRunEntity : - datastore.prepare(q).asIterable(FetchOptions.Builder.withLimit(MAX_RUN_COUNT))) { - TestRunEntity testRun = TestRunEntity.fromEntity(testRunEntity); - if (testRun == null) continue; - testRuns.add(testRun); - } - return testRuns; - } - - /** - * Separate the test cases which are acknowledged by the provided acknowledgments. - * - * @param testCases The list of test case names. - * @param devices The list of devices for a test run. - * @param acks The list of acknowledgments for the test. - * @return A list of acknowledged test case names that have been removed from the input test - * cases. - */ - public static Set<String> separateAcknowledged( - Set<String> testCases, - List<DeviceInfoEntity> devices, - List<TestAcknowledgmentEntity> acks) { - Set<String> acknowledged = new HashSet<>(); - for (TestAcknowledgmentEntity ack : acks) { - boolean allDevices = ack.getDevices() == null || ack.getDevices().size() == 0; - boolean allBranches = ack.getBranches() == null || ack.getBranches().size() == 0; - boolean isRelevant = allDevices && allBranches; - - // Determine if the acknowledgment is relevant to the devices. - if (!isRelevant) { - for (DeviceInfoEntity device : devices) { - boolean deviceAcknowledged = - allDevices || ack.getDevices().contains(device.getBuildFlavor()); - boolean branchAcknowledged = - allBranches || ack.getBranches().contains(device.getBranch()); - if (deviceAcknowledged && branchAcknowledged) isRelevant = true; - } - } - - if (isRelevant) { - // Separate the test cases - boolean allTestCases = - ack.getTestCaseNames() == null || ack.getTestCaseNames().size() == 0; - if (allTestCases) { - acknowledged.addAll(testCases); - testCases.removeAll(acknowledged); - } else { - for (String testCase : ack.getTestCaseNames()) { - if (testCases.contains(testCase)) { - acknowledged.add(testCase); - testCases.remove(testCase); - } - } - } - } - } - return acknowledged; - } - - /** - * Checks whether any new failures have occurred beginning since (and including) startTime. - * - * @param testRuns The list of test runs for which to update the status. - * @param link The string URL linking to the test's status table. - * @param failedTestCaseMap The map of test case names to TestCase for those failing in the last - * status update. - * @param emailAddresses The list of email addresses to send notifications to. - * @param messages The email Message queue. - * @returns latest TestStatusMessage or null if no update is available. - * @throws IOException - */ - public TestStatusEntity getTestStatus( - List<TestRunEntity> testRuns, - String link, - Map<String, TestCase> failedTestCaseMap, - List<TestAcknowledgmentEntity> testAcks, - List<String> emailAddresses, - List<Message> messages) - throws IOException { - if (testRuns.size() == 0) return null; - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - - TestRunEntity mostRecentRun = null; - Map<String, TestCaseResult> mostRecentTestCaseResults = new HashMap<>(); - Map<String, TestCase> testCaseBreakageMap = new HashMap<>(); - int passingTestcaseCount = 0; - List<TestCaseReference> failingTestCases = new ArrayList<>(); - Set<String> fixedTestcases = new HashSet<>(); - Set<String> newTestcaseFailures = new HashSet<>(); - Set<String> continuedTestcaseFailures = new HashSet<>(); - Set<String> skippedTestcaseFailures = new HashSet<>(); - Set<String> transientTestcaseFailures = new HashSet<>(); - - for (TestRunEntity testRun : testRuns) { - if (mostRecentRun == null) { - mostRecentRun = testRun; - } - List<Key> testCaseKeys = new ArrayList<>(); - for (long testCaseId : testRun.getTestCaseIds()) { - testCaseKeys.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId)); - } - Map<Key, Entity> entityMap = datastore.get(testCaseKeys); - for (Key testCaseKey : testCaseKeys) { - if (!entityMap.containsKey(testCaseKey)) { - logger.log(Level.WARNING, "Test case entity missing: " + testCaseKey); - continue; - } - Entity testCaseRun = entityMap.get(testCaseKey); - TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun); - if (testCaseRunEntity == null) { - logger.log(Level.WARNING, "Invalid test case run: " + testCaseRun.getKey()); - continue; - } - for (TestCase testCase : testCaseRunEntity.testCases) { - String testCaseName = testCase.name; - TestCaseResult result = TestCaseResult.valueOf(testCase.result); - - if (mostRecentRun == testRun) { - mostRecentTestCaseResults.put(testCaseName, result); - } else { - if (!mostRecentTestCaseResults.containsKey(testCaseName)) { - // Deprecate notifications for tests that are not present on newer runs - continue; - } - TestCaseResult mostRecentRes = mostRecentTestCaseResults.get(testCaseName); - if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_SKIP) { - mostRecentTestCaseResults.put(testCaseName, result); - } else if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_PASS) { - // Test is passing now, witnessed a transient failure - if (result != TestCaseResult.TEST_CASE_RESULT_PASS - && result != TestCaseResult.TEST_CASE_RESULT_SKIP) { - transientTestcaseFailures.add(testCaseName); - } - } - } - - // Record test case breakages - if (result != TestCaseResult.TEST_CASE_RESULT_PASS - && result != TestCaseResult.TEST_CASE_RESULT_SKIP) { - testCaseBreakageMap.put(testCaseName, testCase); - } - } - } - } - - Set<String> buildIdList = new HashSet<>(); - List<DeviceInfoEntity> devices = new ArrayList<>(); - Query deviceQuery = new Query(DeviceInfoEntity.KIND).setAncestor(mostRecentRun.getKey()); - for (Entity device : datastore.prepare(deviceQuery).asIterable()) { - DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device); - if (deviceEntity == null) { - continue; - } - buildIdList.add(deviceEntity.getBuildId()); - devices.add(deviceEntity); - } - String footer = EmailHelper.getEmailFooter(mostRecentRun, devices, link); - String buildId = StringUtils.join(buildIdList, ","); - - for (String testCaseName : mostRecentTestCaseResults.keySet()) { - TestCaseResult mostRecentResult = mostRecentTestCaseResults.get(testCaseName); - boolean previouslyFailed = failedTestCaseMap.containsKey(testCaseName); - if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_SKIP) { - // persist previous status - if (previouslyFailed) { - skippedTestcaseFailures.add(testCaseName); - failingTestCases.add( - new TestCaseReference(failedTestCaseMap.get(testCaseName))); - } else { - ++passingTestcaseCount; - } - } else if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_PASS) { - ++passingTestcaseCount; - if (previouslyFailed && !transientTestcaseFailures.contains(testCaseName)) { - fixedTestcases.add(testCaseName); - } - } else { - if (!previouslyFailed) { - newTestcaseFailures.add(testCaseName); - failingTestCases.add( - new TestCaseReference(testCaseBreakageMap.get(testCaseName))); - } else { - continuedTestcaseFailures.add(testCaseName); - failingTestCases.add( - new TestCaseReference(failedTestCaseMap.get(testCaseName))); - } - } - } - - Set<String> acknowledgedFailures = - separateAcknowledged(newTestcaseFailures, devices, testAcks); - acknowledgedFailures.addAll( - separateAcknowledged(transientTestcaseFailures, devices, testAcks)); - acknowledgedFailures.addAll( - separateAcknowledged(continuedTestcaseFailures, devices, testAcks)); - - String summary = new String(); - if (newTestcaseFailures.size() + continuedTestcaseFailures.size() > 0) { - summary += "The following test cases failed in the latest test run:<br>"; - - // Add new test case failures to top of summary in bold font. - List<String> sortedNewTestcaseFailures = new ArrayList<>(newTestcaseFailures); - sortedNewTestcaseFailures.sort(Comparator.naturalOrder()); - for (String testcaseName : sortedNewTestcaseFailures) { - summary += "- " + "<b>" + testcaseName + "</b><br>"; - } - - // Add continued test case failures to summary. - List<String> sortedContinuedTestcaseFailures = - new ArrayList<>(continuedTestcaseFailures); - sortedContinuedTestcaseFailures.sort(Comparator.naturalOrder()); - for (String testcaseName : sortedContinuedTestcaseFailures) { - summary += "- " + testcaseName + "<br>"; - } - } - if (fixedTestcases.size() > 0) { - // Add fixed test cases to summary. - summary += "<br><br>The following test cases were fixed in the latest test run:<br>"; - List<String> sortedFixedTestcases = new ArrayList<>(fixedTestcases); - sortedFixedTestcases.sort(Comparator.naturalOrder()); - for (String testcaseName : sortedFixedTestcases) { - summary += "- <i>" + testcaseName + "</i><br>"; - } - } - if (transientTestcaseFailures.size() > 0) { - // Add transient test case failures to summary. - summary += "<br><br>The following transient test case failures occured:<br>"; - List<String> sortedTransientTestcaseFailures = - new ArrayList<>(transientTestcaseFailures); - sortedTransientTestcaseFailures.sort(Comparator.naturalOrder()); - for (String testcaseName : sortedTransientTestcaseFailures) { - summary += "- " + testcaseName + "<br>"; - } - } - if (skippedTestcaseFailures.size() > 0) { - // Add skipped test case failures to summary. - summary += "<br><br>The following test cases have not been run since failing:<br>"; - List<String> sortedSkippedTestcaseFailures = new ArrayList<>(skippedTestcaseFailures); - sortedSkippedTestcaseFailures.sort(Comparator.naturalOrder()); - for (String testcaseName : sortedSkippedTestcaseFailures) { - summary += "- " + testcaseName + "<br>"; - } - } - if (acknowledgedFailures.size() > 0) { - // Add acknowledged test case failures to summary. - List<String> sortedAcknowledgedFailures = new ArrayList<>(acknowledgedFailures); - sortedAcknowledgedFailures.sort(Comparator.naturalOrder()); - if (acknowledgedFailures.size() > 0) { - summary += - "<br><br>The following acknowledged test case failures continued to fail:<br>"; - for (String testcaseName : sortedAcknowledgedFailures) { - summary += "- " + testcaseName + "<br>"; - } - } - } - - String testName = mostRecentRun.getKey().getParent().getName(); - String uploadDateString = TimeUtil.getDateString(mostRecentRun.getStartTimestamp()); - String subject = "VTS Test Alert: " + testName + " @ " + uploadDateString; - if (newTestcaseFailures.size() > 0) { - String body = - "Hello,<br><br>New test case failure(s) in " - + testName - + " for device build ID(s): " - + buildId - + ".<br><br>" - + summary - + footer; - try { - messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); - } catch (MessagingException | UnsupportedEncodingException e) { - logger.log(Level.WARNING, "Error composing email : ", e); - } - } else if (continuedTestcaseFailures.size() > 0) { - String body = - "Hello,<br><br>Continuous test case failure(s) in " - + testName - + " for device build ID(s): " - + buildId - + ".<br><br>" - + summary - + footer; - try { - messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); - } catch (MessagingException | UnsupportedEncodingException e) { - logger.log(Level.WARNING, "Error composing email : ", e); - } - } else if (transientTestcaseFailures.size() > 0) { - String body = - "Hello,<br><br>Transient test case failure(s) in " - + testName - + " but tests all " - + "are passing in the latest device build(s): " - + buildId - + ".<br><br>" - + summary - + footer; - try { - messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); - } catch (MessagingException | UnsupportedEncodingException e) { - logger.log(Level.WARNING, "Error composing email : ", e); - } - } else if (fixedTestcases.size() > 0) { - String body = - "Hello,<br><br>All test cases passed in " - + testName - + " for device build ID(s): " - + buildId - + "!<br><br>" - + summary - + footer; - try { - messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); - } catch (MessagingException | UnsupportedEncodingException e) { - logger.log(Level.WARNING, "Error composing email : ", e); - } - } - return new TestStatusEntity( - testName, - mostRecentRun.getStartTimestamp(), - passingTestcaseCount, - failingTestCases.size(), - failingTestCases); - } - - /** - * Add a task to process test run data - * - * @param testRunKey The key of the test run whose data process. - */ - public static void addTask(Key testRunKey) { - Queue queue = QueueFactory.getDefaultQueue(); - String keyString = KeyFactory.keyToString(testRunKey); - queue.add( - TaskOptions.Builder.withUrl(ALERT_JOB_URL) - .param("runKey", keyString) - .method(TaskOptions.Method.POST)); - } - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws IOException { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - String runKeyString = request.getParameter("runKey"); - - Key testRunKey; - try { - testRunKey = KeyFactory.stringToKey(runKeyString); - } catch (IllegalArgumentException e) { - logger.log(Level.WARNING, "Invalid key specified: " + runKeyString); - return; - } - String testName = testRunKey.getParent().getName(); - - TestStatusEntity status = null; - Key statusKey = KeyFactory.createKey(TestStatusEntity.KIND, testName); - try { - status = TestStatusEntity.fromEntity(datastore.get(statusKey)); - } catch (EntityNotFoundException e) { - // no existing status - } - if (status == null) { - status = new TestStatusEntity(testName); - } - if (status.getUpdatedTimestamp() >= testRunKey.getId()) { - // Another job has already updated the status first - return; - } - List<String> emails = EmailHelper.getSubscriberEmails(testRunKey.getParent()); - - StringBuffer fullUrl = request.getRequestURL(); - String baseUrl = fullUrl.substring(0, fullUrl.indexOf(request.getRequestURI())); - String link = - baseUrl + "/show_tree?testName=" + testName + "&endTime=" + testRunKey.getId(); - - List<Message> messageQueue = new ArrayList<>(); - Map<String, TestCase> failedTestcaseMap = getCurrentFailures(status); - List<TestAcknowledgmentEntity> testAcks = - getTestCaseAcknowledgments(testRunKey.getParent()); - List<TestRunEntity> testRuns = - getTestRuns( - testRunKey.getParent(), status.getUpdatedTimestamp(), testRunKey.getId()); - if (testRuns.size() == 0) return; - - TestStatusEntity newStatus = - getTestStatus(testRuns, link, failedTestcaseMap, testAcks, emails, messageQueue); - if (newStatus == null) { - // No changes to status - return; - } - - int retries = 0; - while (true) { - Transaction txn = datastore.beginTransaction(); - try { - try { - status = TestStatusEntity.fromEntity(datastore.get(statusKey)); - } catch (EntityNotFoundException e) { - // no status left - } - if (status == null - || status.getUpdatedTimestamp() >= newStatus.getUpdatedTimestamp()) { - txn.rollback(); - } else { // This update is most recent. - datastore.put(newStatus.toEntity()); - txn.commit(); - EmailHelper.sendAll(messageQueue); - } - break; - } catch (ConcurrentModificationException - | DatastoreFailureException - | DatastoreTimeoutException e) { - logger.log(Level.WARNING, "Retrying alert job insert: " + statusKey); - if (retries++ >= DatastoreHelper.MAX_WRITE_RETRIES) { - logger.log(Level.SEVERE, "Exceeded alert job retries: " + statusKey); - throw e; - } - } finally { - if (txn.isActive()) { - txn.rollback(); - } - } - } - } -} diff --git a/src/main/java/com/android/vts/job/VtsCoverageAlertJobServlet.java b/src/main/java/com/android/vts/job/VtsCoverageAlertJobServlet.java deleted file mode 100644 index b5453b7..0000000 --- a/src/main/java/com/android/vts/job/VtsCoverageAlertJobServlet.java +++ /dev/null @@ -1,328 +0,0 @@ -/* - * Copyright (c) 2017 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may - * obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.android.vts.job; - -import static com.googlecode.objectify.ObjectifyService.ofy; - -import com.android.vts.entity.CodeCoverageEntity; -import com.android.vts.entity.DeviceInfoEntity; -import com.android.vts.entity.TestCoverageStatusEntity; -import com.android.vts.entity.TestRunEntity; -import com.android.vts.util.EmailHelper; -import com.google.appengine.api.datastore.DatastoreService; -import com.google.appengine.api.datastore.DatastoreServiceFactory; -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityNotFoundException; -import com.google.appengine.api.datastore.Key; -import com.google.appengine.api.datastore.KeyFactory; -import com.google.appengine.api.datastore.Query; -import com.google.appengine.api.taskqueue.Queue; -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TaskOptions; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.math.RoundingMode; -import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.mail.Message; -import javax.mail.MessagingException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import org.apache.commons.lang.StringUtils; - -/** - * Coverage notification job. - */ -public class VtsCoverageAlertJobServlet extends BaseJobServlet { - - private static final String COVERAGE_ALERT_URL = "/task/vts_coverage_job"; - protected static final Logger logger = - Logger.getLogger(VtsCoverageAlertJobServlet.class.getName()); - protected static final double CHANGE_ALERT_THRESHOLD = 0.05; - protected static final double GOOD_THRESHOLD = 0.7; - protected static final double BAD_THRESHOLD = 0.3; - - protected static final DecimalFormat FORMATTER; - - /** Initialize the decimal formatter. */ - static { - FORMATTER = new DecimalFormat("#.#"); - FORMATTER.setRoundingMode(RoundingMode.HALF_UP); - } - - /** - * Gets a new coverage status and adds notification emails to the messages list. - * - * Send an email to notify subscribers in the event that a test goes up or down by more than 5%, - * becomes higher or lower than 70%, or becomes higher or lower than 30%. - * - * @param status The TestCoverageStatusEntity object for the test. - * @param testRunKey The key for TestRunEntity whose data to process and reflect in the state. - * @param link The string URL linking to the test's status table. - * @param emailAddresses The list of email addresses to send notifications to. - * @param messages The email Message queue. - * @returns TestCoverageStatusEntity or null if no update is available. - */ - public static TestCoverageStatusEntity getTestCoverageStatus( - TestCoverageStatusEntity status, - Key testRunKey, - String link, - List<String> emailAddresses, - List<Message> messages) - throws IOException { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - - String testName = status.getTestName(); - - double previousPct; - double coveragePct; - if (status == null || status.getTotalLineCount() <= 0 || status.getCoveredLineCount() < 0) { - previousPct = 0; - } else { - previousPct = ((double) status.getCoveredLineCount()) / status.getTotalLineCount(); - } - - Entity testRun; - try { - testRun = datastore.get(testRunKey); - } catch (EntityNotFoundException e) { - logger.log(Level.WARNING, "Test run not found: " + testRunKey); - return null; - } - - TestRunEntity testRunEntity = TestRunEntity.fromEntity(testRun); - if (testRunEntity == null || !testRunEntity.getHasCodeCoverage()) { - return null; - } - CodeCoverageEntity codeCoverageEntity = testRunEntity.getCodeCoverageEntity(); - - if (codeCoverageEntity.getTotalLineCount() <= 0 - || codeCoverageEntity.getCoveredLineCount() < 0) { - coveragePct = 0; - } else { - coveragePct = - ((double) codeCoverageEntity.getCoveredLineCount()) - / codeCoverageEntity.getTotalLineCount(); - } - - Set<String> buildIdList = new HashSet<>(); - Query deviceQuery = new Query(DeviceInfoEntity.KIND).setAncestor(testRun.getKey()); - List<DeviceInfoEntity> devices = new ArrayList<>(); - for (Entity device : datastore.prepare(deviceQuery).asIterable()) { - DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device); - if (deviceEntity == null) { - continue; - } - devices.add(deviceEntity); - buildIdList.add(deviceEntity.getBuildId()); - } - String deviceBuild = StringUtils.join(buildIdList, ", "); - String footer = EmailHelper.getEmailFooter(testRunEntity, devices, link); - - String subject = null; - String body = null; - String subjectSuffix = " @ " + deviceBuild; - if (coveragePct >= GOOD_THRESHOLD && previousPct < GOOD_THRESHOLD) { - // Coverage entered the good zone - subject = - "Congratulations! " - + testName - + " has exceeded " - + FORMATTER.format(GOOD_THRESHOLD * 100) - + "% coverage" - + subjectSuffix; - body = - "Hello,<br><br>The " - + testName - + " has achieved " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } else if (coveragePct < GOOD_THRESHOLD && previousPct >= GOOD_THRESHOLD) { - // Coverage dropped out of the good zone - subject = - "Warning! " - + testName - + " has dropped below " - + FORMATTER.format(GOOD_THRESHOLD * 100) - + "% coverage" - + subjectSuffix; - ; - body = - "Hello,<br><br>The test " - + testName - + " has dropped to " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } else if (coveragePct <= BAD_THRESHOLD && previousPct > BAD_THRESHOLD) { - // Coverage entered into the bad zone - subject = - "Warning! " - + testName - + " has dropped below " - + FORMATTER.format(BAD_THRESHOLD * 100) - + "% coverage" - + subjectSuffix; - body = - "Hello,<br><br>The test " - + testName - + " has dropped to " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } else if (coveragePct > BAD_THRESHOLD && previousPct <= BAD_THRESHOLD) { - // Coverage emerged from the bad zone - subject = - "Congratulations! " - + testName - + " has exceeded " - + FORMATTER.format(BAD_THRESHOLD * 100) - + "% coverage" - + subjectSuffix; - body = - "Hello,<br><br>The test " - + testName - + " has achived " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } else if (coveragePct - previousPct < -CHANGE_ALERT_THRESHOLD) { - // Send a coverage drop alert - subject = - "Warning! " - + testName - + "'s code coverage has decreased by more than " - + FORMATTER.format(CHANGE_ALERT_THRESHOLD * 100) - + "%" - + subjectSuffix; - body = - "Hello,<br><br>The test " - + testName - + " has dropped from " - + FORMATTER.format(previousPct * 100) - + "% code coverage to " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } else if (coveragePct - previousPct > CHANGE_ALERT_THRESHOLD) { - // Send a coverage improvement alert - subject = - testName - + "'s code coverage has increased by more than " - + FORMATTER.format(CHANGE_ALERT_THRESHOLD * 100) - + "%" - + subjectSuffix; - body = - "Hello,<br><br>The test " - + testName - + " has increased from " - + FORMATTER.format(previousPct * 100) - + "% code coverage to " - + FORMATTER.format(coveragePct * 100) - + "% code coverage on device build ID(s): " - + deviceBuild - + "." - + footer; - } - if (subject != null && body != null) { - try { - messages.add(EmailHelper.composeEmail(emailAddresses, subject, body)); - } catch (MessagingException | UnsupportedEncodingException e) { - logger.log(Level.WARNING, "Error composing email : ", e); - } - } - return new TestCoverageStatusEntity( - testName, - testRunEntity.getStartTimestamp(), - codeCoverageEntity.getCoveredLineCount(), - codeCoverageEntity.getTotalLineCount(), - devices.size() > 0 ? devices.get(0).getId() : 0); - } - - /** - * Add a task to process coverage data - * - * @param testRunKey The key of the test run whose data process. - */ - public static void addTask(Key testRunKey) { - Queue queue = QueueFactory.getDefaultQueue(); - String keyString = KeyFactory.keyToString(testRunKey); - queue.add( - TaskOptions.Builder.withUrl(COVERAGE_ALERT_URL) - .param("runKey", keyString) - .method(TaskOptions.Method.POST)); - } - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws IOException { - String runKeyString = request.getParameter("runKey"); - - Key testRunKey; - try { - testRunKey = KeyFactory.stringToKey(runKeyString); - } catch (IllegalArgumentException e) { - logger.log(Level.WARNING, "Invalid key specified: " + runKeyString); - return; - } - String testName = testRunKey.getParent().getName(); - - TestCoverageStatusEntity status = ofy().load().type(TestCoverageStatusEntity.class).id(testName) - .now(); - if (status == null) { - status = new TestCoverageStatusEntity(testName, 0, -1, -1, 0); - } - - StringBuffer fullUrl = request.getRequestURL(); - String baseUrl = fullUrl.substring(0, fullUrl.indexOf(request.getRequestURI())); - String link = baseUrl + "/show_tree?testName=" + testName; - TestCoverageStatusEntity newStatus; - List<Message> messageQueue = new ArrayList<>(); - try { - List<String> emails = EmailHelper.getSubscriberEmails(testRunKey.getParent()); - newStatus = getTestCoverageStatus(status, testRunKey, link, emails, messageQueue); - } catch (IOException e) { - logger.log(Level.SEVERE, e.toString()); - return; - } - - if (newStatus == null) { - return; - } else { - if (status == null || status.getUpdatedTimestamp() < newStatus.getUpdatedTimestamp()) { - newStatus.save(); - EmailHelper.sendAll(messageQueue); - } - } - } -} diff --git a/src/main/java/com/android/vts/job/VtsInactivityJobServlet.java b/src/main/java/com/android/vts/job/VtsInactivityJobServlet.java deleted file mode 100644 index 031cdbb..0000000 --- a/src/main/java/com/android/vts/job/VtsInactivityJobServlet.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright (c) 2017 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may - * obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.android.vts.job; - -import com.android.vts.entity.TestEntity; -import com.android.vts.entity.TestRunEntity; -import com.android.vts.entity.TestStatusEntity; -import com.android.vts.util.EmailHelper; -import com.android.vts.util.FilterUtil; -import com.android.vts.util.TaskQueueHelper; -import com.android.vts.util.TimeUtil; -import com.google.appengine.api.datastore.DatastoreService; -import com.google.appengine.api.datastore.DatastoreServiceFactory; -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityNotFoundException; -import com.google.appengine.api.datastore.FetchOptions; -import com.google.appengine.api.datastore.Key; -import com.google.appengine.api.datastore.KeyFactory; -import com.google.appengine.api.datastore.Query; -import com.google.appengine.api.datastore.Query.Filter; -import com.google.appengine.api.datastore.Query.SortDirection; -import com.google.appengine.api.taskqueue.Queue; -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TaskOptions; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.mail.Message; -import javax.mail.MessagingException; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/** Test inactivity notification job. */ -public class VtsInactivityJobServlet extends BaseJobServlet { - private static final String INACTIVITY_ALERT_URL = "/cron/vts_inactivity_job"; - protected static final Logger logger = - Logger.getLogger(VtsInactivityJobServlet.class.getName()); - - /** - * Compose an email if the test is inactive. - * - * @param test The TestStatusEntity document storing the test status. - * @param lastRunTime The timestamp in microseconds of the last test run for this test. - * @param link Fully specified link to the test's status page. - * @param emails The list of email addresses to send the email. - * @param messages The message list in which to insert the inactivity notification email. - * @return True if the test is inactive, false otherwise. - */ - private static boolean notifyIfInactive( - TestStatusEntity test, - long lastRunTime, - String link, - List<String> emails, - List<Message> messages) { - long now = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis()); - long diff = now - lastRunTime; - // Send an email daily to notify that the test hasn't been running. - // After 7 full days have passed, notifications will no longer be sent (i.e. the - // test is assumed to be deprecated). - if (diff >= TimeUnit.DAYS.toMicros(1) && diff < TimeUnit.DAYS.toMicros(8)) { - String uploadTimeString = TimeUtil.getDateTimeZoneString(lastRunTime); - String subject = "Warning! Inactive test: " + test.getTestName(); - String body = - "Hello,<br><br>Test \"" - + test.getTestName() - + "\" is inactive. " - + "No new data has been uploaded since " - + uploadTimeString - + "." - + EmailHelper.getEmailFooter(null, null, link); - try { - messages.add(EmailHelper.composeEmail(emails, subject, body)); - return true; - } catch (MessagingException | UnsupportedEncodingException e) { - logger.log(Level.WARNING, "Error composing email : ", e); - } - } - return false; - } - - /** - * Get the timestamp for the last test run for the specified test. - * - * @param testKey The parent key of the test runs to query for. - * @return The timestamp in microseconds of the last test run for the test, or -1 if none. - */ - private static long getLastRunTime(Key testKey) { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - Filter testTypeFilter = FilterUtil.getTestTypeFilter(false, true, false); - Query q = - new Query(TestRunEntity.KIND) - .setAncestor(testKey) - .setFilter(testTypeFilter) - .addSort(Entity.KEY_RESERVED_PROPERTY, SortDirection.DESCENDING) - .setKeysOnly(); - - long lastTestRun = -1; - for (Entity testRun : datastore.prepare(q).asIterable(FetchOptions.Builder.withLimit(1))) { - lastTestRun = testRun.getKey().getId(); - } - return lastTestRun; - } - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - Queue queue = QueueFactory.getDefaultQueue(); - Query q = new Query(TestStatusEntity.KIND).setKeysOnly(); - List<TaskOptions> tasks = new ArrayList<>(); - for (Entity status : datastore.prepare(q).asIterable()) { - TaskOptions task = - TaskOptions.Builder.withUrl(INACTIVITY_ALERT_URL) - .param("statusKey", KeyFactory.keyToString(status.getKey())) - .method(TaskOptions.Method.POST); - tasks.add(task); - } - TaskQueueHelper.addToQueue(queue, tasks); - } - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws IOException { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - String statusKeyString = request.getParameter("statusKey"); - - Key statusKey; - try { - statusKey = KeyFactory.stringToKey(statusKeyString); - } catch (IllegalArgumentException e) { - logger.log(Level.WARNING, "Invalid key specified: " + statusKeyString); - return; - } - - TestStatusEntity status = null; - try { - status = TestStatusEntity.fromEntity(datastore.get(statusKey)); - } catch (EntityNotFoundException e) { - // no existing status - } - if (status == null) { - return; - } - Key testKey = KeyFactory.createKey(TestEntity.KIND, status.getTestName()); - long lastRunTime = getLastRunTime(testKey); - - StringBuffer fullUrl = request.getRequestURL(); - String baseUrl = fullUrl.substring(0, fullUrl.indexOf(request.getRequestURI())); - String link = baseUrl + "/show_tree?testName=" + status.getTestName(); - - List<Message> messageQueue = new ArrayList<>(); - List<String> emails; - try { - emails = EmailHelper.getSubscriberEmails(testKey); - } catch (IOException e) { - logger.log(Level.SEVERE, e.toString()); - return; - } - notifyIfInactive(status, lastRunTime, link, emails, messageQueue); - if (messageQueue.size() > 0) { - EmailHelper.sendAll(messageQueue); - } - } -} diff --git a/src/main/java/com/android/vts/job/VtsPerformanceJobServlet.java b/src/main/java/com/android/vts/job/VtsPerformanceJobServlet.java deleted file mode 100644 index ebd5294..0000000 --- a/src/main/java/com/android/vts/job/VtsPerformanceJobServlet.java +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Copyright (c) 2016 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may - * obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.android.vts.job; - -import com.android.vts.entity.TestEntity; -import com.android.vts.util.EmailHelper; -import com.android.vts.util.PerformanceSummary; -import com.android.vts.util.PerformanceUtil; -import com.android.vts.util.ProfilingPointSummary; -import com.android.vts.util.StatSummary; -import com.android.vts.util.TaskQueueHelper; -import com.google.appengine.api.datastore.DatastoreService; -import com.google.appengine.api.datastore.DatastoreServiceFactory; -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.Key; -import com.google.appengine.api.datastore.KeyFactory; -import com.google.appengine.api.datastore.Query; -import com.google.appengine.api.taskqueue.Queue; -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TaskOptions; -import java.io.IOException; -import java.math.RoundingMode; -import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/** Represents the notifications service which is automatically called on a fixed schedule. */ -public class VtsPerformanceJobServlet extends BaseJobServlet { - protected static final Logger logger = - Logger.getLogger(VtsPerformanceJobServlet.class.getName()); - - private static final String PERFORMANCE_JOB_URL = "/cron/vts_performance_job"; - private static final String MEAN = "Mean"; - private static final String MAX = "Max"; - private static final String MIN = "Min"; - private static final String MIN_DELTA = "ΔMin (%)"; - private static final String MAX_DELTA = "ΔMax (%)"; - private static final String HIGHER_IS_BETTER = - "Note: Higher values are better. Maximum is the best-case performance."; - private static final String LOWER_IS_BETTER = - "Note: Lower values are better. Minimum is the best-case performance."; - private static final String STD = "Std"; - private static final String SUBJECT_PREFIX = "Daily Performance Digest: "; - private static final String LAST_WEEK = "Last Week"; - private static final String LABEL_STYLE = "font-family: arial"; - private static final String SUBTEXT_STYLE = "font-family: arial; font-size: 12px"; - private static final String TABLE_STYLE = - "width: 100%; border-collapse: collapse; border: 1px solid black; font-size: 12px; font-family: arial;"; - private static final String SECTION_LABEL_STYLE = - "border: 1px solid black; border-bottom: none; background-color: lightgray;"; - private static final String COL_LABEL_STYLE = - "border: 1px solid black; border-bottom-width: 2px; border-top: 1px dotted gray; background-color: lightgray;"; - private static final String HEADER_COL_STYLE = - "border-top: 1px dotted gray; border-right: 2px solid black; text-align: right; background-color: lightgray;"; - private static final String INNER_CELL_STYLE = - "border-top: 1px dotted gray; border-right: 1px dotted gray; text-align: right;"; - private static final String OUTER_CELL_STYLE = - "border-top: 1px dotted gray; border-right: 2px solid black; text-align: right;"; - - private static final DecimalFormat FORMATTER; - - /** Initialize the decimal formatter. */ - static { - FORMATTER = new DecimalFormat("#.##"); - FORMATTER.setRoundingMode(RoundingMode.HALF_UP); - } - - /** - * Generates an HTML summary of the performance changes for the profiling results in the - * specified table. - * - * <p>Retrieves the past 24 hours of profiling data and compares it to the 24 hours that - * preceded it. Creates a table representation of the mean and standard deviation for each - * profiling point. When performance degrades, the cell is shaded red. - * - * @param testName The name of the test whose profiling data to summarize. - * @param perfSummaries List of PerformanceSummary objects for each profiling run (in reverse - * chronological order). - * @param labels List of string labels for use as the column headers. - * @returns An HTML string containing labeled table summaries. - */ - public static String getPerformanceSummary( - String testName, List<PerformanceSummary> perfSummaries, List<String> labels) { - if (perfSummaries.size() == 0) return ""; - PerformanceSummary now = perfSummaries.get(0); - String tableHTML = "<p style='" + LABEL_STYLE + "'><b>"; - tableHTML += testName + "</b></p>"; - for (String profilingPoint : now.getProfilingPointNames()) { - ProfilingPointSummary summary = now.getProfilingPointSummary(profilingPoint); - tableHTML += "<table cellpadding='2' style='" + TABLE_STYLE + "'>"; - - // Format header rows - String[] headerRows = new String[] {profilingPoint, summary.yLabel}; - int colspan = labels.size() * 4; - for (String content : headerRows) { - tableHTML += "<tr><td colspan='" + colspan + "'>" + content + "</td></tr>"; - } - - // Format section labels - tableHTML += "<tr>"; - for (int i = 0; i < labels.size(); i++) { - String content = labels.get(i); - tableHTML += "<th style='" + SECTION_LABEL_STYLE + "' "; - if (i == 0) tableHTML += "colspan='1'"; - else if (i == 1) tableHTML += "colspan='3'"; - else tableHTML += "colspan='4'"; - tableHTML += ">" + content + "</th>"; - } - tableHTML += "</tr>"; - - String deltaString; - String bestCaseString; - String subtext; - switch (now.getProfilingPointSummary(profilingPoint).getRegressionMode()) { - case VTS_REGRESSION_MODE_DECREASING: - deltaString = MAX_DELTA; - bestCaseString = MAX; - subtext = HIGHER_IS_BETTER; - break; - default: - deltaString = MIN_DELTA; - bestCaseString = MIN; - subtext = LOWER_IS_BETTER; - break; - } - - // Format column labels - tableHTML += "<tr>"; - for (int i = 0; i < labels.size(); i++) { - if (i > 1) { - tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + deltaString + "</th>"; - } - if (i == 0) { - tableHTML += "<th style='" + COL_LABEL_STYLE + "'>"; - tableHTML += summary.xLabel + "</th>"; - } else if (i > 0) { - tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + bestCaseString + "</th>"; - tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + MEAN + "</th>"; - tableHTML += "<th style='" + COL_LABEL_STYLE + "'>" + STD + "</th>"; - } - } - tableHTML += "</tr>"; - - // Populate data cells - for (StatSummary stats : summary) { - String label = stats.getLabel(); - tableHTML += "<tr><td style='" + HEADER_COL_STYLE + "'>" + label; - tableHTML += "</td><td style='" + INNER_CELL_STYLE + "'>"; - tableHTML += FORMATTER.format(stats.getBestCase()) + "</td>"; - tableHTML += "<td style='" + INNER_CELL_STYLE + "'>"; - tableHTML += FORMATTER.format(stats.getMean()) + "</td>"; - tableHTML += "<td style='" + OUTER_CELL_STYLE + "'>"; - if (stats.getCount() < 2) { - tableHTML += " - </td>"; - } else { - tableHTML += FORMATTER.format(stats.getStd()) + "</td>"; - } - for (int i = 1; i < perfSummaries.size(); i++) { - PerformanceSummary oldPerfSummary = perfSummaries.get(i); - if (oldPerfSummary.hasProfilingPoint(profilingPoint)) { - StatSummary baseline = - oldPerfSummary - .getProfilingPointSummary(profilingPoint) - .getStatSummary(label); - tableHTML += - PerformanceUtil.getBestCasePerformanceComparisonHTML( - baseline, - stats, - "", - "", - INNER_CELL_STYLE, - OUTER_CELL_STYLE); - } else tableHTML += "<td></td><td></td><td></td><td></td>"; - } - tableHTML += "</tr>"; - } - tableHTML += "</table>"; - tableHTML += "<i style='" + SUBTEXT_STYLE + "'>" + subtext + "</i><br><br>"; - } - return tableHTML; - } - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - Queue queue = QueueFactory.getDefaultQueue(); - Query q = new Query(TestEntity.KIND).setKeysOnly(); - List<TaskOptions> tasks = new ArrayList<>(); - for (Entity test : datastore.prepare(q).asIterable()) { - if (test.getKey().getName() == null) { - continue; - } - TaskOptions task = - TaskOptions.Builder.withUrl(PERFORMANCE_JOB_URL) - .param("testKey", KeyFactory.keyToString(test.getKey())) - .method(TaskOptions.Method.POST); - tasks.add(task); - } - TaskQueueHelper.addToQueue(queue, tasks); - } - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws IOException { - String testKeyString = request.getParameter("testKey"); - Key testKey; - try { - testKey = KeyFactory.stringToKey(testKeyString); - } catch (IllegalArgumentException e) { - logger.log(Level.WARNING, "Invalid key specified: " + testKeyString); - return; - } - - long nowMicro = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis()); - - // Add today to the list of time intervals to analyze - List<PerformanceSummary> summaries = new ArrayList<>(); - PerformanceSummary today = - new PerformanceSummary(nowMicro - TimeUnit.DAYS.toMicros(1), nowMicro); - summaries.add(today); - - // Add yesterday as a baseline time interval for analysis - long oneDayAgo = nowMicro - TimeUnit.DAYS.toMicros(1); - PerformanceSummary yesterday = - new PerformanceSummary(oneDayAgo - TimeUnit.DAYS.toMicros(1), oneDayAgo); - summaries.add(yesterday); - - // Add last week as a baseline time interval for analysis - long oneWeek = TimeUnit.DAYS.toMicros(7); - long oneWeekAgo = nowMicro - oneWeek; - - String spanString = "<span class='date-label'>"; - String label = - spanString + TimeUnit.MICROSECONDS.toMillis(oneWeekAgo - oneWeek) + "</span>"; - label += " - " + spanString + TimeUnit.MICROSECONDS.toMillis(oneWeekAgo) + "</span>"; - PerformanceSummary lastWeek = - new PerformanceSummary(oneWeekAgo - oneWeek, oneWeekAgo, label); - summaries.add(lastWeek); - PerformanceUtil.updatePerformanceSummary( - testKey.getName(), oneWeekAgo - oneWeek, nowMicro, null, summaries); - - List<PerformanceSummary> nonEmptySummaries = new ArrayList<>(); - List<String> labels = new ArrayList<>(); - labels.add(""); - for (PerformanceSummary perfSummary : summaries) { - if (perfSummary.size() == 0) continue; - nonEmptySummaries.add(perfSummary); - labels.add(perfSummary.label); - } - String body = getPerformanceSummary(testKey.getName(), nonEmptySummaries, labels); - if (body == null || body.equals("")) { - return; - } - List<String> emails = EmailHelper.getSubscriberEmails(testKey); - if (emails.size() == 0) { - return; - } - String subject = SUBJECT_PREFIX + testKey.getName(); - EmailHelper.send(emails, subject, body); - } -} diff --git a/src/main/java/com/android/vts/job/VtsProfilingStatsJobServlet.java b/src/main/java/com/android/vts/job/VtsProfilingStatsJobServlet.java deleted file mode 100644 index 50f72e1..0000000 --- a/src/main/java/com/android/vts/job/VtsProfilingStatsJobServlet.java +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Copyright (C) 2017 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may - * obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.android.vts.job; - -import com.android.vts.entity.DeviceInfoEntity; -import com.android.vts.entity.ProfilingPointEntity; -import com.android.vts.entity.ProfilingPointRunEntity; -import com.android.vts.entity.ProfilingPointSummaryEntity; -import com.android.vts.util.DatastoreHelper; -import com.android.vts.util.PerformanceUtil; -import com.android.vts.util.TaskQueueHelper; -import com.android.vts.util.TimeUtil; -import com.google.appengine.api.datastore.DatastoreFailureException; -import com.google.appengine.api.datastore.DatastoreService; -import com.google.appengine.api.datastore.DatastoreServiceFactory; -import com.google.appengine.api.datastore.DatastoreTimeoutException; -import com.google.appengine.api.datastore.Entity; -import com.google.appengine.api.datastore.EntityNotFoundException; -import com.google.appengine.api.datastore.Key; -import com.google.appengine.api.datastore.KeyFactory; -import com.google.appengine.api.datastore.Query; -import com.google.appengine.api.datastore.Transaction; -import com.google.appengine.api.taskqueue.Queue; -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TaskOptions; -import java.io.IOException; -import java.time.Instant; -import java.time.ZonedDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.ConcurrentModificationException; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -/** Represents the notifications service which is automatically called on a fixed schedule. */ -public class VtsProfilingStatsJobServlet extends BaseJobServlet { - protected static final Logger logger = - Logger.getLogger(VtsProfilingStatsJobServlet.class.getName()); - private static final String HIDL_HAL_OPTION = "hidl_hal_mode"; - private static final String[] splitKeysArray = new String[] {HIDL_HAL_OPTION}; - private static final Set<String> splitKeySet = new HashSet<>(Arrays.asList(splitKeysArray)); - - public static final String PROFILING_STATS_JOB_URL = "/task/vts_profiling_stats_job"; - public static final String PROFILING_POINT_KEY = "profilingPointKey"; - public static final String QUEUE = "profilingStatsQueue"; - - /** - * Round the date down to the start of the day (PST). - * - * @param time The time in microseconds. - * @return - */ - public static long getCanonicalTime(long time) { - long timeMillis = TimeUnit.MICROSECONDS.toMillis(time); - ZonedDateTime zdt = - ZonedDateTime.ofInstant(Instant.ofEpochMilli(timeMillis), TimeUtil.PT_ZONE); - return TimeUnit.SECONDS.toMicros( - zdt.withHour(0).withMinute(0).withSecond(0).toEpochSecond()); - } - - /** - * Add tasks to process profiling run data - * - * @param profilingPointKeys The list of keys of the profiling point runs whose data process. - */ - public static void addTasks(List<Key> profilingPointKeys) { - Queue queue = QueueFactory.getQueue(QUEUE); - List<TaskOptions> tasks = new ArrayList<>(); - for (Key key : profilingPointKeys) { - String keyString = KeyFactory.keyToString(key); - tasks.add( - TaskOptions.Builder.withUrl(PROFILING_STATS_JOB_URL) - .param(PROFILING_POINT_KEY, keyString) - .method(TaskOptions.Method.POST)); - } - TaskQueueHelper.addToQueue(queue, tasks); - } - - /** - * Update the profiling summaries with the information from a profiling point run. - * - * @param testKey The key to the TestEntity whose profiling data to analyze. - * @param profilingPointRun The profiling data to analyze. - * @param devices The list of devices used in the profiling run. - * @param time The canonical timestamp of the summary to update. - * @return true if the update succeeds, false otherwise. - */ - public static boolean updateSummaries( - Key testKey, - ProfilingPointRunEntity profilingPointRun, - List<DeviceInfoEntity> devices, - long time) { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - Transaction tx = datastore.beginTransaction(); - try { - List<Entity> puts = new ArrayList<>(); - - ProfilingPointEntity profilingPoint = - new ProfilingPointEntity( - testKey.getName(), - profilingPointRun.getName(), - profilingPointRun.getType(), - profilingPointRun.getRegressionMode(), - profilingPointRun.getXLabel(), - profilingPointRun.getYLabel()); - puts.add(profilingPoint.toEntity()); - - String option = PerformanceUtil.getOptionAlias(profilingPointRun, splitKeySet); - - Set<String> branches = new HashSet<>(); - Set<String> deviceNames = new HashSet<>(); - - branches.add(ProfilingPointSummaryEntity.ALL); - deviceNames.add(ProfilingPointSummaryEntity.ALL); - - for (DeviceInfoEntity d : devices) { - branches.add(d.getBranch()); - deviceNames.add(d.getBuildFlavor()); - } - - List<Key> summaryGets = new ArrayList<>(); - for (String branch : branches) { - for (String device : deviceNames) { - summaryGets.add( - ProfilingPointSummaryEntity.createKey( - profilingPoint.getKey(), branch, device, option, time)); - } - } - - Map<Key, Entity> summaries = datastore.get(tx, summaryGets); - Map<String, Map<String, ProfilingPointSummaryEntity>> summaryMap = new HashMap<>(); - for (Key key : summaries.keySet()) { - Entity e = summaries.get(key); - ProfilingPointSummaryEntity profilingPointSummary = - ProfilingPointSummaryEntity.fromEntity(e); - if (profilingPointSummary == null) { - logger.log(Level.WARNING, "Invalid profiling point summary: " + e.getKey()); - continue; - } - if (!summaryMap.containsKey(profilingPointSummary.getBranch())) { - summaryMap.put(profilingPointSummary.getBranch(), new HashMap<>()); - } - Map<String, ProfilingPointSummaryEntity> deviceMap = - summaryMap.get(profilingPointSummary.getBranch()); - deviceMap.put(profilingPointSummary.getBuildFlavor(), profilingPointSummary); - } - - Set<ProfilingPointSummaryEntity> modifiedEntities = new HashSet<>(); - - for (String branch : branches) { - if (!summaryMap.containsKey(branch)) { - summaryMap.put(branch, new HashMap<>()); - } - Map<String, ProfilingPointSummaryEntity> deviceMap = summaryMap.get(branch); - - for (String device : deviceNames) { - ProfilingPointSummaryEntity summary; - if (deviceMap.containsKey(device)) { - summary = deviceMap.get(device); - } else { - summary = - new ProfilingPointSummaryEntity( - profilingPoint.getKey(), branch, device, option, time); - deviceMap.put(device, summary); - } - summary.update(profilingPointRun); - modifiedEntities.add(summary); - } - } - - for (ProfilingPointSummaryEntity profilingPointSummary : modifiedEntities) { - puts.add(profilingPointSummary.toEntity()); - } - datastore.put(tx, puts); - tx.commit(); - } catch (ConcurrentModificationException - | DatastoreFailureException - | DatastoreTimeoutException e) { - return false; - } finally { - if (tx.isActive()) { - tx.rollback(); - logger.log( - Level.WARNING, - "Profiling stats job transaction still active: " - + profilingPointRun.getKey()); - return false; - } - } - return true; - } - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws IOException { - DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - String profilingPointKeyString = request.getParameter(PROFILING_POINT_KEY); - - Key profilingPointRunKey; - try { - profilingPointRunKey = KeyFactory.stringToKey(profilingPointKeyString); - } catch (IllegalArgumentException e) { - logger.log(Level.WARNING, "Invalid key specified: " + profilingPointKeyString); - return; - } - Key testKey = profilingPointRunKey.getParent().getParent(); - - ProfilingPointRunEntity profilingPointRun = null; - try { - Entity profilingPointRunEntity = datastore.get(profilingPointRunKey); - profilingPointRun = ProfilingPointRunEntity.fromEntity(profilingPointRunEntity); - } catch (EntityNotFoundException e) { - // no run found - } - if (profilingPointRun == null) { - return; - } - - Query deviceQuery = - new Query(DeviceInfoEntity.KIND).setAncestor(profilingPointRunKey.getParent()); - - List<DeviceInfoEntity> devices = new ArrayList<>(); - for (Entity e : datastore.prepare(deviceQuery).asIterable()) { - DeviceInfoEntity deviceInfoEntity = DeviceInfoEntity.fromEntity(e); - if (e == null) continue; - devices.add(deviceInfoEntity); - } - - long canonicalTime = getCanonicalTime(profilingPointRunKey.getParent().getId()); - int retryCount = 0; - while (retryCount++ <= DatastoreHelper.MAX_WRITE_RETRIES) { - boolean result = updateSummaries(testKey, profilingPointRun, devices, canonicalTime); - if (!result) { - logger.log( - Level.WARNING, "Retrying profiling stats update: " + profilingPointRunKey); - continue; - } - break; - } - if (retryCount > DatastoreHelper.MAX_WRITE_RETRIES) { - logger.log(Level.SEVERE, "Could not update profiling stats: " + profilingPointRunKey); - } - } -} diff --git a/src/main/java/com/android/vts/job/VtsSpreadSheetSyncServlet.java b/src/main/java/com/android/vts/job/VtsSpreadSheetSyncServlet.java deleted file mode 100644 index 45726d9..0000000 --- a/src/main/java/com/android/vts/job/VtsSpreadSheetSyncServlet.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (C) 2018 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may - * obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.android.vts.job; - -import com.android.vts.entity.ApiCoverageExcludedEntity; -import com.android.vts.entity.DashboardEntity; -import com.google.api.client.auth.oauth2.Credential; -import com.google.api.client.extensions.appengine.datastore.AppEngineDataStoreFactory; -import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp; -import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver; -import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow; -import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets; -import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; -import com.google.api.client.http.javanet.NetHttpTransport; -import com.google.api.client.json.JsonFactory; -import com.google.api.client.json.jackson2.JacksonFactory; -import com.google.api.services.sheets.v4.Sheets; -import com.google.api.services.sheets.v4.model.ValueRange; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.security.GeneralSecurityException; -import java.util.ArrayList; -import java.util.List; - -import javax.servlet.ServletConfig; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; - -/** Job to sync excluded API data in google spreadsheet with datastore's entity. */ -public class VtsSpreadSheetSyncServlet extends BaseJobServlet { - - protected static final Logger logger = - Logger.getLogger(VtsSpreadSheetSyncServlet.class.getName()); - - private static final String APPLICATION_NAME = "VTS Dashboard"; - private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance(); - - private String CREDENTIALS_KEY_FILE = ""; - - /** GoogleClientSecrets for GoogleAuthorizationCodeFlow Builder */ - private GoogleClientSecrets clientSecrets; - - /** This is the ID of google spreadsheet. */ - private String SPREAD_SHEET_ID = ""; - - /** This is the range to read of google spreadsheet. */ - private String SPREAD_SHEET_RANGE = ""; - - @Override - public void init(ServletConfig servletConfig) throws ServletException { - super.init(servletConfig); - - try { - CREDENTIALS_KEY_FILE = systemConfigProp.getProperty("api.coverage.keyFile"); - SPREAD_SHEET_ID = systemConfigProp.getProperty("api.coverage.spreadSheetId"); - SPREAD_SHEET_RANGE = systemConfigProp.getProperty("api.coverage.spreadSheetRange"); - - InputStream keyFileInputStream = - this.getClass() - .getClassLoader() - .getResourceAsStream("keys/" + CREDENTIALS_KEY_FILE); - InputStreamReader keyFileStreamReader = new InputStreamReader(keyFileInputStream); - - this.clientSecrets = GoogleClientSecrets.load(JSON_FACTORY, keyFileStreamReader); - } catch (IOException ioe) { - logger.log(Level.SEVERE, ioe.getMessage()); - } catch (Exception exception) { - logger.log(Level.SEVERE, exception.getMessage()); - } - } - - /** - * Creates an authorized Credential object. - * - * @param HTTP_TRANSPORT The network HTTP Transport. - * @param appEngineDataStoreFactory The credential will be persisted using the Google App Engine - * Data Store API. - * @param SCOPES Scopes are strings that enable access to particular resources, such as user - * data. - * @return An authorized Credential object. - * @throws IOException If the credentials.json file cannot be found. - */ - private Credential getCredentials( - final NetHttpTransport HTTP_TRANSPORT, - final AppEngineDataStoreFactory appEngineDataStoreFactory, - final List<String> SCOPES) - throws IOException { - - // Build flow and trigger user authorization request. - GoogleAuthorizationCodeFlow flow = - new GoogleAuthorizationCodeFlow.Builder( - HTTP_TRANSPORT, JSON_FACTORY, this.clientSecrets, SCOPES) - .setDataStoreFactory(appEngineDataStoreFactory) - .setAccessType("offline") - .build(); - LocalServerReceiver localServerReceiver = new LocalServerReceiver(); - return new AuthorizationCodeInstalledApp(flow, localServerReceiver).authorize("user"); - } - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { - - try { - // Build a new authorized API client service. - final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport(); - final AppEngineDataStoreFactory appEngineDataStoreFactory = - (AppEngineDataStoreFactory) - request.getServletContext().getAttribute("dataStoreFactory"); - final List<String> googleApiScopes = - (List<String>) request.getServletContext().getAttribute("googleApiScopes"); - - Sheets service = - new Sheets.Builder( - HTTP_TRANSPORT, - JSON_FACTORY, - getCredentials( - HTTP_TRANSPORT, - appEngineDataStoreFactory, - googleApiScopes)) - .setApplicationName(APPLICATION_NAME) - .build(); - - ValueRange valueRange = - service.spreadsheets() - .values() - .get(SPREAD_SHEET_ID, SPREAD_SHEET_RANGE) - .execute(); - - List<ApiCoverageExcludedEntity> apiCoverageExcludedEntities = new ArrayList<>(); - List<List<Object>> values = valueRange.getValues(); - if (values == null || values.isEmpty()) { - logger.log(Level.WARNING, "No data found in google spreadsheet."); - } else { - for (List row : values) { - ApiCoverageExcludedEntity apiCoverageExcludedEntity = - new ApiCoverageExcludedEntity( - row.get(0).toString(), - row.get(1).toString(), - row.get(2).toString(), - row.get(3).toString(), - row.get(4).toString()); - apiCoverageExcludedEntities.add(apiCoverageExcludedEntity); - } - } - - DashboardEntity.saveAll(apiCoverageExcludedEntities, MAX_ENTITY_SIZE_PER_TRANSACTION); - - } catch (GeneralSecurityException gse) { - logger.log(Level.SEVERE, gse.getMessage()); - } - } -} diff --git a/src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java b/src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java deleted file mode 100644 index 2aa34e4..0000000 --- a/src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright (c) 2018 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may - * obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.android.vts.job; - -import com.android.vts.entity.TestSuiteFileEntity; -import com.android.vts.entity.TestSuiteResultEntity; -import com.android.vts.proto.TestSuiteResultMessageProto; -import com.android.vts.util.GcsHelper; -import com.android.vts.util.TaskQueueHelper; -import com.android.vts.util.TimeUtil; -import com.google.appengine.api.memcache.ErrorHandlers; -import com.google.appengine.api.memcache.MemcacheService; -import com.google.appengine.api.memcache.MemcacheServiceFactory; -import com.google.appengine.api.taskqueue.Queue; -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TaskOptions; -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.Bucket; -import com.google.cloud.storage.Storage; -import com.googlecode.objectify.Key; -import javax.servlet.ServletConfig; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.FileSystems; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static com.googlecode.objectify.ObjectifyService.ofy; - -/** Suite Test result file processing job. */ -public class VtsSuiteTestJobServlet extends BaseJobServlet { - - private static final String SUITE_TEST_URL = "/cron/test_suite_report_gcs_monitor"; - - private static final String SERVICE_NAME = "VTS Dashboard"; - - private final Logger logger = Logger.getLogger(this.getClass().getName()); - - /** Google Cloud Storage project's key file to access the storage */ - private static String GCS_KEY_FILE; - /** Google Cloud Storage project's default bucket name for vtslab log files */ - private static String GCS_BUCKET_NAME; - /** Google Cloud Storage project's default directory name for suite test result files */ - private static String GCS_SUITE_TEST_FOLDER_NAME; - - public static final String QUEUE = "suiteTestQueue"; - - /** - * This is the key file to access vtslab-gcs project. It will allow the dashboard to have a full - * control of the bucket. - */ - private InputStream keyFileInputStream; - - /** This is the instance of java google storage library */ - private Storage storage; - - /** This is the instance of App Engine memcache service java library */ - private MemcacheService syncCache = MemcacheServiceFactory.getMemcacheService(); - - @Override - public void init(ServletConfig servletConfig) throws ServletException { - super.init(servletConfig); - - GCS_KEY_FILE = systemConfigProp.getProperty("gcs.keyFile"); - GCS_BUCKET_NAME = systemConfigProp.getProperty("gcs.bucketName"); - GCS_SUITE_TEST_FOLDER_NAME = systemConfigProp.getProperty("gcs.suiteTestFolderName"); - - this.keyFileInputStream = - this.getClass().getClassLoader().getResourceAsStream("keys/" + GCS_KEY_FILE); - - Optional<Storage> optionalStorage = GcsHelper.getStorage(this.keyFileInputStream); - if (optionalStorage.isPresent()) { - this.storage = optionalStorage.get(); - } else { - logger.log(Level.SEVERE, "Error on getting storage instance!"); - throw new ServletException("Creating storage instance exception!"); - } - syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO)); - } - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { - - List<String> dateStringList = new ArrayList<>(); - - long currentMicroSecond = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis()); - - ZonedDateTime checkZonedDateTime = TimeUtil.getZonedDateTime(currentMicroSecond); - String checkDateString = - DateTimeFormatter.ofPattern(TimeUtil.DATE_FORMAT) - .format(checkZonedDateTime.minusMinutes(5)); - String todayDateString = TimeUtil.getDateString(currentMicroSecond); - if (!checkDateString.equals(todayDateString)) { - dateStringList.add(checkDateString); - logger.log(Level.INFO, "Yesterday is added to the process queue and processed!"); - } - dateStringList.add(todayDateString); - - for (String dateString : dateStringList) { - String[] dateArray = dateString.split("-"); - if (dateArray.length == 3) { - - Queue queue = QueueFactory.getQueue(QUEUE); - - List<TaskOptions> tasks = new ArrayList<>(); - - String fileSeparator = FileSystems.getDefault().getSeparator(); - - String year = dateArray[0]; - String month = dateArray[1]; - String day = dateArray[2]; - - List<String> pathList = Arrays.asList(GCS_SUITE_TEST_FOLDER_NAME, year, month, day); - Path pathInfo = Paths.get(String.join(fileSeparator, pathList)); - - List<TestSuiteFileEntity> testSuiteFileEntityList = - ofy().load() - .type(TestSuiteFileEntity.class) - .filter("year", Integer.parseInt(year)) - .filter("month", Integer.parseInt(month)) - .filter("day", Integer.parseInt(day)) - .list(); - - List<String> filePathList = - testSuiteFileEntityList - .stream() - .map(testSuiteFile -> testSuiteFile.getFilePath()) - .collect(Collectors.toList()); - - Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME); - - Storage.BlobListOption[] listOptions = - new Storage.BlobListOption[] { - Storage.BlobListOption.prefix(pathInfo.toString() + fileSeparator) - }; - - Iterable<Blob> blobIterable = vtsReportBucket.list(listOptions).iterateAll(); - Iterator<Blob> blobIterator = blobIterable.iterator(); - while (blobIterator.hasNext()) { - Blob blob = blobIterator.next(); - if (blob.isDirectory()) { - logger.log(Level.INFO, blob.getName() + " directory will be skipped!"); - } else { - if (filePathList.contains(blob.getName())) { - logger.log(Level.INFO, "filePathList contain => " + blob.getName()); - } else if (blob.getName().endsWith(fileSeparator)) { - logger.log(Level.INFO, blob.getName() + " endswith slash!"); - } else { - TaskOptions task = - TaskOptions.Builder.withUrl(SUITE_TEST_URL) - .param("filePath", blob.getName()) - .method(TaskOptions.Method.POST); - tasks.add(task); - } - } - } - TaskQueueHelper.addToQueue(queue, tasks); - } else { - throw new IllegalArgumentException( - todayDateString + " date string not in correct format"); - } - } - } - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws IOException { - - String filePath = request.getParameter("filePath"); - if (Objects.isNull(filePath)) { - logger.log(Level.WARNING, "filePath parameter is null!"); - } else { - logger.log(Level.INFO, "filePath parameter => " + filePath); - - Key<TestSuiteFileEntity> testSuiteFileEntityKey = - Key.create(TestSuiteFileEntity.class, filePath); - TestSuiteFileEntity testSuiteFileEntity = - ofy().load() - .type(TestSuiteFileEntity.class) - .filterKey(testSuiteFileEntityKey) - .first() - .now(); - - if (Objects.isNull(testSuiteFileEntity)) { - Blob blobFile = (Blob) this.syncCache.get(filePath); - if (Objects.isNull(blobFile)) { - Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME); - blobFile = vtsReportBucket.get(filePath); - this.syncCache.put(filePath, blobFile); - } - - if (blobFile.exists()) { - TestSuiteFileEntity newTestSuiteFileEntity = new TestSuiteFileEntity(filePath); - try { - TestSuiteResultMessageProto.TestSuiteResultMessage testSuiteResultMessage = - TestSuiteResultMessageProto.TestSuiteResultMessage.parseFrom( - blobFile.getContent()); - - TestSuiteResultEntity testSuiteResultEntity = - new TestSuiteResultEntity( - testSuiteFileEntityKey, - testSuiteResultMessage.getStartTime(), - testSuiteResultMessage.getEndTime(), - testSuiteResultMessage.getTestType(), - testSuiteResultMessage.getBootSuccess(), - testSuiteResultMessage.getResultPath(), - testSuiteResultMessage.getInfraLogPath(), - testSuiteResultMessage.getHostName(), - testSuiteResultMessage.getSuitePlan(), - testSuiteResultMessage.getSuiteVersion(), - testSuiteResultMessage.getSuiteName(), - testSuiteResultMessage.getSuiteBuildNumber(), - testSuiteResultMessage.getModulesDone(), - testSuiteResultMessage.getModulesTotal(), - testSuiteResultMessage.getBranch(), - testSuiteResultMessage.getTarget(), - testSuiteResultMessage.getBuildId(), - testSuiteResultMessage.getBuildSystemFingerprint(), - testSuiteResultMessage.getBuildVendorFingerprint(), - testSuiteResultMessage.getPassedTestCaseCount(), - testSuiteResultMessage.getFailedTestCaseCount()); - - testSuiteResultEntity.save(newTestSuiteFileEntity); - } catch (IOException e) { - ofy().delete().type(TestSuiteFileEntity.class).id(filePath).now(); - response.setStatus(HttpServletResponse.SC_BAD_REQUEST); - logger.log(Level.WARNING, "Invalid proto: " + e.getLocalizedMessage()); - } - } - } else { - logger.log(Level.INFO, "suite test found in datastore!"); - } - } - } -} |