path: root/src/main/java/com/android/vts/job/VtsAlertJobServlet.java

/*
 * Copyright (c) 2016 Google Inc. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License"); you
 * may not use this file except in compliance with the License. You may
 * obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 * implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */

package com.android.vts.job;

import com.android.vts.entity.DeviceInfoEntity;
import com.android.vts.entity.TestAcknowledgmentEntity;
import com.android.vts.entity.TestCaseRunEntity;
import com.android.vts.entity.TestCaseRunEntity.TestCase;
import com.android.vts.entity.TestRunEntity;
import com.android.vts.entity.TestStatusEntity;
import com.android.vts.entity.TestStatusEntity.TestCaseReference;
import com.android.vts.proto.VtsReportMessage.TestCaseResult;
import com.android.vts.util.DatastoreHelper;
import com.android.vts.util.EmailHelper;
import com.android.vts.util.FilterUtil;
import com.android.vts.util.TimeUtil;
import com.google.appengine.api.datastore.DatastoreFailureException;
import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.DatastoreServiceFactory;
import com.google.appengine.api.datastore.DatastoreTimeoutException;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.EntityNotFoundException;
import com.google.appengine.api.datastore.FetchOptions;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.datastore.KeyFactory;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.Query.Filter;
import com.google.appengine.api.datastore.Query.SortDirection;
import com.google.appengine.api.datastore.Transaction;
import com.google.appengine.api.taskqueue.Queue;
import com.google.appengine.api.taskqueue.QueueFactory;
import com.google.appengine.api.taskqueue.TaskOptions;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.ConcurrentModificationException;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.logging.Level;
import java.util.logging.Logger;
import javax.mail.Message;
import javax.mail.MessagingException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import org.apache.commons.lang.StringUtils;

/** Represents the notifications service which is automatically called on a fixed schedule. */
public class VtsAlertJobServlet extends BaseJobServlet {
    private static final String ALERT_JOB_URL = "/task/vts_alert_job";
    protected static final Logger logger = Logger.getLogger(VtsAlertJobServlet.class.getName());
    protected static final int MAX_RUN_COUNT = 1000; // maximum number of runs to query for

    /**
     * Process the current test case failures for a test.
     *
     * @param status The TestStatusEntity object for the test.
     * @returns a map from test case name to the test case run ID for which the test case failed.
     */
    private static Map<String, TestCase> getCurrentFailures(TestStatusEntity status) {
        if (status.getFailingTestCases() == null || status.getFailingTestCases().size() == 0) {
            return new HashMap<>();
        }
        DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
        Map<String, TestCase> failingTestcases = new HashMap<>();
        Set<Key> gets = new HashSet<>();
        for (TestCaseReference testCaseRef : status.getFailingTestCases()) {
            gets.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId));
        }
        if (gets.size() == 0) {
            return failingTestcases;
        }
        Map<Key, Entity> testCaseMap = datastore.get(gets);

        for (TestCaseReference testCaseRef : status.getFailingTestCases()) {
            Key key = KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseRef.parentId);
            if (!testCaseMap.containsKey(key)) {
                continue;
            }
            Entity testCaseRun = testCaseMap.get(key);
            TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun);
            if (testCaseRunEntity.testCases.size() <= testCaseRef.offset) {
                continue;
            }
            TestCase testCase = testCaseRunEntity.testCases.get(testCaseRef.offset);
            failingTestcases.put(testCase.name, testCase);
        }
        return failingTestcases;
    }

    /**
     * Get the test acknowledgments for a test key.
     *
     * @param testKey The key to the test whose acknowledgments to fetch.
     * @return A list of test acknowledgments.
     */
    private static List<TestAcknowledgmentEntity> getTestCaseAcknowledgments(Key testKey) {
        DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();

        List<TestAcknowledgmentEntity> acks = new ArrayList<>();
        Filter testFilter =
                new Query.FilterPredicate(
                        TestAcknowledgmentEntity.TEST_KEY, Query.FilterOperator.EQUAL, testKey);
        Query q = new Query(TestAcknowledgmentEntity.KIND).setFilter(testFilter);

        for (Entity ackEntity : datastore.prepare(q).asIterable()) {
            TestAcknowledgmentEntity ack = TestAcknowledgmentEntity.fromEntity(ackEntity);
            if (ack == null) continue;
            acks.add(ack);
        }
        return acks;
    }

    /**
     * Get the test runs for the test in the specified time window.
     *
     * <p>If the start and end time delta is greater than one day, the query will be truncated.
     *
     * @param testKey The key to the test whose runs to query.
     * @param startTime The start time for the query.
     * @param endTime The end time for the query.
     * @return A list of test runs in the specified time window.
     */
    private static List<TestRunEntity> getTestRuns(Key testKey, long startTime, long endTime) {
        DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
        Filter testTypeFilter = FilterUtil.getTestTypeFilter(false, true, false);
        long delta = endTime - startTime;
        delta = Math.min(delta, TimeUnit.DAYS.toMicros(1));
        Filter runFilter =
                FilterUtil.getTimeFilter(
                        testKey, TestRunEntity.KIND, endTime - delta + 1, endTime, testTypeFilter);

        Query q =
                new Query(TestRunEntity.KIND)
                        .setAncestor(testKey)
                        .setFilter(runFilter)
                        .addSort(Entity.KEY_RESERVED_PROPERTY, SortDirection.DESCENDING);

        List<TestRunEntity> testRuns = new ArrayList<>();
        for (Entity testRunEntity :
                datastore.prepare(q).asIterable(FetchOptions.Builder.withLimit(MAX_RUN_COUNT))) {
            TestRunEntity testRun = TestRunEntity.fromEntity(testRunEntity);
            if (testRun == null) continue;
            testRuns.add(testRun);
        }
        return testRuns;
    }

    /**
     * Separate the test cases which are acknowledged by the provided acknowledgments.
     *
     * @param testCases The list of test case names.
     * @param devices The list of devices for a test run.
     * @param acks The list of acknowledgments for the test.
     * @return A list of acknowledged test case names that have been removed from the input test
     *     cases.
     */
    public static Set<String> separateAcknowledged(
            Set<String> testCases,
            List<DeviceInfoEntity> devices,
            List<TestAcknowledgmentEntity> acks) {
        Set<String> acknowledged = new HashSet<>();
        for (TestAcknowledgmentEntity ack : acks) {
            boolean allDevices = ack.getDevices() == null || ack.getDevices().size() == 0;
            boolean allBranches = ack.getBranches() == null || ack.getBranches().size() == 0;
            boolean isRelevant = allDevices && allBranches;

            // Determine if the acknowledgment is relevant to the devices.
            if (!isRelevant) {
                for (DeviceInfoEntity device : devices) {
                    boolean deviceAcknowledged =
                            allDevices || ack.getDevices().contains(device.getBuildFlavor());
                    boolean branchAcknowledged =
                            allBranches || ack.getBranches().contains(device.getBranch());
                    if (deviceAcknowledged && branchAcknowledged) isRelevant = true;
                }
            }

            if (isRelevant) {
                // Separate the test cases
                boolean allTestCases =
                        ack.getTestCaseNames() == null || ack.getTestCaseNames().size() == 0;
                if (allTestCases) {
                    acknowledged.addAll(testCases);
                    testCases.removeAll(acknowledged);
                } else {
                    for (String testCase : ack.getTestCaseNames()) {
                        if (testCases.contains(testCase)) {
                            acknowledged.add(testCase);
                            testCases.remove(testCase);
                        }
                    }
                }
            }
        }
        return acknowledged;
    }

    /**
     * Checks whether any new failures have occurred beginning since (and including) startTime.
     *
     * @param testRuns The list of test runs for which to update the status.
     * @param link The string URL linking to the test's status table.
     * @param failedTestCaseMap The map of test case names to TestCase for those failing in the last
     *     status update.
     * @param emailAddresses The list of email addresses to send notifications to.
     * @param messages The email Message queue.
     * @returns latest TestStatusMessage or null if no update is available.
     * @throws IOException
     */
    public TestStatusEntity getTestStatus(
            List<TestRunEntity> testRuns,
            String link,
            Map<String, TestCase> failedTestCaseMap,
            List<TestAcknowledgmentEntity> testAcks,
            List<String> emailAddresses,
            List<Message> messages)
            throws IOException {
        if (testRuns.size() == 0) return null;
        DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();

        TestRunEntity mostRecentRun = null;
        Map<String, TestCaseResult> mostRecentTestCaseResults = new HashMap<>();
        Map<String, TestCase> testCaseBreakageMap = new HashMap<>();
        int passingTestcaseCount = 0;
        List<TestCaseReference> failingTestCases = new ArrayList<>();
        Set<String> fixedTestcases = new HashSet<>();
        Set<String> newTestcaseFailures = new HashSet<>();
        Set<String> continuedTestcaseFailures = new HashSet<>();
        Set<String> skippedTestcaseFailures = new HashSet<>();
        Set<String> transientTestcaseFailures = new HashSet<>();

        for (TestRunEntity testRun : testRuns) {
            if (mostRecentRun == null) {
                mostRecentRun = testRun;
            }
            List<Key> testCaseKeys = new ArrayList<>();
            for (long testCaseId : testRun.getTestCaseIds()) {
                testCaseKeys.add(KeyFactory.createKey(TestCaseRunEntity.KIND, testCaseId));
            }
            Map<Key, Entity> entityMap = datastore.get(testCaseKeys);
            for (Key testCaseKey : testCaseKeys) {
                if (!entityMap.containsKey(testCaseKey)) {
                    logger.log(Level.WARNING, "Test case entity missing: " + testCaseKey);
                    continue;
                }
                Entity testCaseRun = entityMap.get(testCaseKey);
                TestCaseRunEntity testCaseRunEntity = TestCaseRunEntity.fromEntity(testCaseRun);
                if (testCaseRunEntity == null) {
                    logger.log(Level.WARNING, "Invalid test case run: " + testCaseRun.getKey());
                    continue;
                }
                for (TestCase testCase : testCaseRunEntity.testCases) {
                    String testCaseName = testCase.name;
                    TestCaseResult result = TestCaseResult.valueOf(testCase.result);

                    if (mostRecentRun == testRun) {
                        mostRecentTestCaseResults.put(testCaseName, result);
                    } else {
                        if (!mostRecentTestCaseResults.containsKey(testCaseName)) {
                            // Deprecate notifications for tests that are not present on newer runs
                            continue;
                        }
                        TestCaseResult mostRecentRes = mostRecentTestCaseResults.get(testCaseName);
                        if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_SKIP) {
                            mostRecentTestCaseResults.put(testCaseName, result);
                        } else if (mostRecentRes == TestCaseResult.TEST_CASE_RESULT_PASS) {
                            // Test is passing now, witnessed a transient failure
                            if (result != TestCaseResult.TEST_CASE_RESULT_PASS
                                    && result != TestCaseResult.TEST_CASE_RESULT_SKIP) {
                                transientTestcaseFailures.add(testCaseName);
                            }
                        }
                    }

                    // Record test case breakages
                    if (result != TestCaseResult.TEST_CASE_RESULT_PASS
                            && result != TestCaseResult.TEST_CASE_RESULT_SKIP) {
                        testCaseBreakageMap.put(testCaseName, testCase);
                    }
                }
            }
        }

        Set<String> buildIdList = new HashSet<>();
        List<DeviceInfoEntity> devices = new ArrayList<>();
        Query deviceQuery = new Query(DeviceInfoEntity.KIND).setAncestor(mostRecentRun.getKey());
        for (Entity device : datastore.prepare(deviceQuery).asIterable()) {
            DeviceInfoEntity deviceEntity = DeviceInfoEntity.fromEntity(device);
            if (deviceEntity == null) {
                continue;
            }
            buildIdList.add(deviceEntity.getBuildId());
            devices.add(deviceEntity);
        }
        String footer = EmailHelper.getEmailFooter(mostRecentRun, devices, link);
        String buildId = StringUtils.join(buildIdList, ",");

        for (String testCaseName : mostRecentTestCaseResults.keySet()) {
            TestCaseResult mostRecentResult = mostRecentTestCaseResults.get(testCaseName);
            boolean previouslyFailed = failedTestCaseMap.containsKey(testCaseName);
            if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_SKIP) {
                // persist previous status
                if (previouslyFailed) {
                    skippedTestcaseFailures.add(testCaseName);
                    failingTestCases.add(
                            new TestCaseReference(failedTestCaseMap.get(testCaseName)));
                } else {
                    ++passingTestcaseCount;
                }
            } else if (mostRecentResult == TestCaseResult.TEST_CASE_RESULT_PASS) {
                ++passingTestcaseCount;
                if (previouslyFailed && !transientTestcaseFailures.contains(testCaseName)) {
                    fixedTestcases.add(testCaseName);
                }
            } else {
                if (!previouslyFailed) {
                    newTestcaseFailures.add(testCaseName);
                    failingTestCases.add(
                            new TestCaseReference(testCaseBreakageMap.get(testCaseName)));
                } else {
                    continuedTestcaseFailures.add(testCaseName);
                    failingTestCases.add(
                            new TestCaseReference(failedTestCaseMap.get(testCaseName)));
                }
            }
        }

        Set<String> acknowledgedFailures =
                separateAcknowledged(newTestcaseFailures, devices, testAcks);
        acknowledgedFailures.addAll(
                separateAcknowledged(transientTestcaseFailures, devices, testAcks));
        acknowledgedFailures.addAll(
                separateAcknowledged(continuedTestcaseFailures, devices, testAcks));

        String summary = new String();
        if (newTestcaseFailures.size() + continuedTestcaseFailures.size() > 0) {
            summary += "The following test cases failed in the latest test run:<br>";

            // Add new test case failures to top of summary in bold font.
            List<String> sortedNewTestcaseFailures = new ArrayList<>(newTestcaseFailures);
            sortedNewTestcaseFailures.sort(Comparator.naturalOrder());
            for (String testcaseName : sortedNewTestcaseFailures) {
                summary += "- " + "<b>" + testcaseName + "</b><br>";
            }

            // Add continued test case failures to summary.
            List<String> sortedContinuedTestcaseFailures =
                    new ArrayList<>(continuedTestcaseFailures);
            sortedContinuedTestcaseFailures.sort(Comparator.naturalOrder());
            for (String testcaseName : sortedContinuedTestcaseFailures) {
                summary += "- " + testcaseName + "<br>";
            }
        }
        if (fixedTestcases.size() > 0) {
            // Add fixed test cases to summary.
            summary += "<br><br>The following test cases were fixed in the latest test run:<br>";
            List<String> sortedFixedTestcases = new ArrayList<>(fixedTestcases);
            sortedFixedTestcases.sort(Comparator.naturalOrder());
            for (String testcaseName : sortedFixedTestcases) {
                summary += "- <i>" + testcaseName + "</i><br>";
            }
        }
        if (transientTestcaseFailures.size() > 0) {
            // Add transient test case failures to summary.
            summary += "<br><br>The following transient test case failures occured:<br>";
            List<String> sortedTransientTestcaseFailures =
                    new ArrayList<>(transientTestcaseFailures);
            sortedTransientTestcaseFailures.sort(Comparator.naturalOrder());
            for (String testcaseName : sortedTransientTestcaseFailures) {
                summary += "- " + testcaseName + "<br>";
            }
        }
        if (skippedTestcaseFailures.size() > 0) {
            // Add skipped test case failures to summary.
            summary += "<br><br>The following test cases have not been run since failing:<br>";
            List<String> sortedSkippedTestcaseFailures = new ArrayList<>(skippedTestcaseFailures);
            sortedSkippedTestcaseFailures.sort(Comparator.naturalOrder());
            for (String testcaseName : sortedSkippedTestcaseFailures) {
                summary += "- " + testcaseName + "<br>";
            }
        }
        if (acknowledgedFailures.size() > 0) {
            // Add acknowledged test case failures to summary.
            List<String> sortedAcknowledgedFailures = new ArrayList<>(acknowledgedFailures);
            sortedAcknowledgedFailures.sort(Comparator.naturalOrder());
            if (acknowledgedFailures.size() > 0) {
                summary +=
                        "<br><br>The following acknowledged test case failures continued to fail:<br>";
                for (String testcaseName : sortedAcknowledgedFailures) {
                    summary += "- " + testcaseName + "<br>";
                }
            }
        }

        String testName = mostRecentRun.getKey().getParent().getName();
        String uploadDateString = TimeUtil.getDateString(mostRecentRun.getStartTimestamp());
        String subject = "VTS Test Alert: " + testName + " @ " + uploadDateString;
        if (newTestcaseFailures.size() > 0) {
            String body =
                    "Hello,<br><br>New test case failure(s) in "
                            + testName
                            + " for device build ID(s): "
                            + buildId
                            + ".<br><br>"
                            + summary
                            + footer;
            try {
                messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
            } catch (MessagingException | UnsupportedEncodingException e) {
                logger.log(Level.WARNING, "Error composing email : ", e);
            }
        } else if (continuedTestcaseFailures.size() > 0) {
            String body =
                    "Hello,<br><br>Continuous test case failure(s) in "
                            + testName
                            + " for device build ID(s): "
                            + buildId
                            + ".<br><br>"
                            + summary
                            + footer;
            try {
                messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
            } catch (MessagingException | UnsupportedEncodingException e) {
                logger.log(Level.WARNING, "Error composing email : ", e);
            }
        } else if (transientTestcaseFailures.size() > 0) {
            String body =
                    "Hello,<br><br>Transient test case failure(s) in "
                            + testName
                            + " but tests all "
                            + "are passing in the latest device build(s): "
                            + buildId
                            + ".<br><br>"
                            + summary
                            + footer;
            try {
                messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
            } catch (MessagingException | UnsupportedEncodingException e) {
                logger.log(Level.WARNING, "Error composing email : ", e);
            }
        } else if (fixedTestcases.size() > 0) {
            String body =
                    "Hello,<br><br>All test cases passed in "
                            + testName
                            + " for device build ID(s): "
                            + buildId
                            + "!<br><br>"
                            + summary
                            + footer;
            try {
                messages.add(EmailHelper.composeEmail(emailAddresses, subject, body));
            } catch (MessagingException | UnsupportedEncodingException e) {
                logger.log(Level.WARNING, "Error composing email : ", e);
            }
        }
        return new TestStatusEntity(
                testName,
                mostRecentRun.getStartTimestamp(),
                passingTestcaseCount,
                failingTestCases.size(),
                failingTestCases);
    }

    /**
     * Add a task to process test run data
     *
     * @param testRunKey The key of the test run whose data process.
     */
    public static void addTask(Key testRunKey) {
        Queue queue = QueueFactory.getDefaultQueue();
        String keyString = KeyFactory.keyToString(testRunKey);
        queue.add(
                TaskOptions.Builder.withUrl(ALERT_JOB_URL)
                        .param("runKey", keyString)
                        .method(TaskOptions.Method.POST));
    }

    @Override
    public void doPost(HttpServletRequest request, HttpServletResponse response)
            throws IOException {
        DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
        String runKeyString = request.getParameter("runKey");

        Key testRunKey;
        try {
            testRunKey = KeyFactory.stringToKey(runKeyString);
        } catch (IllegalArgumentException e) {
            logger.log(Level.WARNING, "Invalid key specified: " + runKeyString);
            return;
        }
        String testName = testRunKey.getParent().getName();

        TestStatusEntity status = null;
        Key statusKey = KeyFactory.createKey(TestStatusEntity.KIND, testName);
        try {
            status = TestStatusEntity.fromEntity(datastore.get(statusKey));
        } catch (EntityNotFoundException e) {
            // no existing status
        }
        if (status == null) {
            status = new TestStatusEntity(testName);
        }
        if (status.getUpdatedTimestamp() >= testRunKey.getId()) {
            // Another job has already updated the status first
            return;
        }
        List<String> emails = EmailHelper.getSubscriberEmails(testRunKey.getParent());

        StringBuffer fullUrl = request.getRequestURL();
        String baseUrl = fullUrl.substring(0, fullUrl.indexOf(request.getRequestURI()));
        String link =
                baseUrl + "/show_tree?testName=" + testName + "&endTime=" + testRunKey.getId();

        List<Message> messageQueue = new ArrayList<>();
        Map<String, TestCase> failedTestcaseMap = getCurrentFailures(status);
        List<TestAcknowledgmentEntity> testAcks =
                getTestCaseAcknowledgments(testRunKey.getParent());
        List<TestRunEntity> testRuns =
                getTestRuns(
                        testRunKey.getParent(), status.getUpdatedTimestamp(), testRunKey.getId());
        if (testRuns.size() == 0) return;

        TestStatusEntity newStatus =
                getTestStatus(testRuns, link, failedTestcaseMap, testAcks, emails, messageQueue);
        if (newStatus == null) {
            // No changes to status
            return;
        }

        int retries = 0;
        while (true) {
            Transaction txn = datastore.beginTransaction();
            try {
                try {
                    status = TestStatusEntity.fromEntity(datastore.get(statusKey));
                } catch (EntityNotFoundException e) {
                    // no status left
                }
                if (status == null
                        || status.getUpdatedTimestamp() >= newStatus.getUpdatedTimestamp()) {
                    txn.rollback();
                } else { // This update is most recent.
                    datastore.put(newStatus.toEntity());
                    txn.commit();
                    EmailHelper.sendAll(messageQueue);
                }
                break;
            } catch (ConcurrentModificationException
                    | DatastoreFailureException
                    | DatastoreTimeoutException e) {
                logger.log(Level.WARNING, "Retrying alert job insert: " + statusKey);
                if (retries++ >= DatastoreHelper.MAX_WRITE_RETRIES) {
                    logger.log(Level.SEVERE, "Exceeded alert job retries: " + statusKey);
                    throw e;
                }
            } finally {
                if (txn.isActive()) {
                    txn.rollback();
                }
            }
        }
    }
}