diff options
Diffstat (limited to 'src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java')
-rw-r--r-- | src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java | 264 |
1 files changed, 0 insertions, 264 deletions
diff --git a/src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java b/src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java deleted file mode 100644 index 2aa34e4..0000000 --- a/src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright (c) 2018 Google Inc. All Rights Reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you - * may not use this file except in compliance with the License. You may - * obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - * implied. See the License for the specific language governing - * permissions and limitations under the License. - */ - -package com.android.vts.job; - -import com.android.vts.entity.TestSuiteFileEntity; -import com.android.vts.entity.TestSuiteResultEntity; -import com.android.vts.proto.TestSuiteResultMessageProto; -import com.android.vts.util.GcsHelper; -import com.android.vts.util.TaskQueueHelper; -import com.android.vts.util.TimeUtil; -import com.google.appengine.api.memcache.ErrorHandlers; -import com.google.appengine.api.memcache.MemcacheService; -import com.google.appengine.api.memcache.MemcacheServiceFactory; -import com.google.appengine.api.taskqueue.Queue; -import com.google.appengine.api.taskqueue.QueueFactory; -import com.google.appengine.api.taskqueue.TaskOptions; -import com.google.cloud.storage.Blob; -import com.google.cloud.storage.Bucket; -import com.google.cloud.storage.Storage; -import com.googlecode.objectify.Key; -import javax.servlet.ServletConfig; -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.FileSystems; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.TimeUnit; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static com.googlecode.objectify.ObjectifyService.ofy; - -/** Suite Test result file processing job. */ -public class VtsSuiteTestJobServlet extends BaseJobServlet { - - private static final String SUITE_TEST_URL = "/cron/test_suite_report_gcs_monitor"; - - private static final String SERVICE_NAME = "VTS Dashboard"; - - private final Logger logger = Logger.getLogger(this.getClass().getName()); - - /** Google Cloud Storage project's key file to access the storage */ - private static String GCS_KEY_FILE; - /** Google Cloud Storage project's default bucket name for vtslab log files */ - private static String GCS_BUCKET_NAME; - /** Google Cloud Storage project's default directory name for suite test result files */ - private static String GCS_SUITE_TEST_FOLDER_NAME; - - public static final String QUEUE = "suiteTestQueue"; - - /** - * This is the key file to access vtslab-gcs project. It will allow the dashboard to have a full - * control of the bucket. - */ - private InputStream keyFileInputStream; - - /** This is the instance of java google storage library */ - private Storage storage; - - /** This is the instance of App Engine memcache service java library */ - private MemcacheService syncCache = MemcacheServiceFactory.getMemcacheService(); - - @Override - public void init(ServletConfig servletConfig) throws ServletException { - super.init(servletConfig); - - GCS_KEY_FILE = systemConfigProp.getProperty("gcs.keyFile"); - GCS_BUCKET_NAME = systemConfigProp.getProperty("gcs.bucketName"); - GCS_SUITE_TEST_FOLDER_NAME = systemConfigProp.getProperty("gcs.suiteTestFolderName"); - - this.keyFileInputStream = - this.getClass().getClassLoader().getResourceAsStream("keys/" + GCS_KEY_FILE); - - Optional<Storage> optionalStorage = GcsHelper.getStorage(this.keyFileInputStream); - if (optionalStorage.isPresent()) { - this.storage = optionalStorage.get(); - } else { - logger.log(Level.SEVERE, "Error on getting storage instance!"); - throw new ServletException("Creating storage instance exception!"); - } - syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO)); - } - - @Override - public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { - - List<String> dateStringList = new ArrayList<>(); - - long currentMicroSecond = TimeUnit.MILLISECONDS.toMicros(System.currentTimeMillis()); - - ZonedDateTime checkZonedDateTime = TimeUtil.getZonedDateTime(currentMicroSecond); - String checkDateString = - DateTimeFormatter.ofPattern(TimeUtil.DATE_FORMAT) - .format(checkZonedDateTime.minusMinutes(5)); - String todayDateString = TimeUtil.getDateString(currentMicroSecond); - if (!checkDateString.equals(todayDateString)) { - dateStringList.add(checkDateString); - logger.log(Level.INFO, "Yesterday is added to the process queue and processed!"); - } - dateStringList.add(todayDateString); - - for (String dateString : dateStringList) { - String[] dateArray = dateString.split("-"); - if (dateArray.length == 3) { - - Queue queue = QueueFactory.getQueue(QUEUE); - - List<TaskOptions> tasks = new ArrayList<>(); - - String fileSeparator = FileSystems.getDefault().getSeparator(); - - String year = dateArray[0]; - String month = dateArray[1]; - String day = dateArray[2]; - - List<String> pathList = Arrays.asList(GCS_SUITE_TEST_FOLDER_NAME, year, month, day); - Path pathInfo = Paths.get(String.join(fileSeparator, pathList)); - - List<TestSuiteFileEntity> testSuiteFileEntityList = - ofy().load() - .type(TestSuiteFileEntity.class) - .filter("year", Integer.parseInt(year)) - .filter("month", Integer.parseInt(month)) - .filter("day", Integer.parseInt(day)) - .list(); - - List<String> filePathList = - testSuiteFileEntityList - .stream() - .map(testSuiteFile -> testSuiteFile.getFilePath()) - .collect(Collectors.toList()); - - Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME); - - Storage.BlobListOption[] listOptions = - new Storage.BlobListOption[] { - Storage.BlobListOption.prefix(pathInfo.toString() + fileSeparator) - }; - - Iterable<Blob> blobIterable = vtsReportBucket.list(listOptions).iterateAll(); - Iterator<Blob> blobIterator = blobIterable.iterator(); - while (blobIterator.hasNext()) { - Blob blob = blobIterator.next(); - if (blob.isDirectory()) { - logger.log(Level.INFO, blob.getName() + " directory will be skipped!"); - } else { - if (filePathList.contains(blob.getName())) { - logger.log(Level.INFO, "filePathList contain => " + blob.getName()); - } else if (blob.getName().endsWith(fileSeparator)) { - logger.log(Level.INFO, blob.getName() + " endswith slash!"); - } else { - TaskOptions task = - TaskOptions.Builder.withUrl(SUITE_TEST_URL) - .param("filePath", blob.getName()) - .method(TaskOptions.Method.POST); - tasks.add(task); - } - } - } - TaskQueueHelper.addToQueue(queue, tasks); - } else { - throw new IllegalArgumentException( - todayDateString + " date string not in correct format"); - } - } - } - - @Override - public void doPost(HttpServletRequest request, HttpServletResponse response) - throws IOException { - - String filePath = request.getParameter("filePath"); - if (Objects.isNull(filePath)) { - logger.log(Level.WARNING, "filePath parameter is null!"); - } else { - logger.log(Level.INFO, "filePath parameter => " + filePath); - - Key<TestSuiteFileEntity> testSuiteFileEntityKey = - Key.create(TestSuiteFileEntity.class, filePath); - TestSuiteFileEntity testSuiteFileEntity = - ofy().load() - .type(TestSuiteFileEntity.class) - .filterKey(testSuiteFileEntityKey) - .first() - .now(); - - if (Objects.isNull(testSuiteFileEntity)) { - Blob blobFile = (Blob) this.syncCache.get(filePath); - if (Objects.isNull(blobFile)) { - Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME); - blobFile = vtsReportBucket.get(filePath); - this.syncCache.put(filePath, blobFile); - } - - if (blobFile.exists()) { - TestSuiteFileEntity newTestSuiteFileEntity = new TestSuiteFileEntity(filePath); - try { - TestSuiteResultMessageProto.TestSuiteResultMessage testSuiteResultMessage = - TestSuiteResultMessageProto.TestSuiteResultMessage.parseFrom( - blobFile.getContent()); - - TestSuiteResultEntity testSuiteResultEntity = - new TestSuiteResultEntity( - testSuiteFileEntityKey, - testSuiteResultMessage.getStartTime(), - testSuiteResultMessage.getEndTime(), - testSuiteResultMessage.getTestType(), - testSuiteResultMessage.getBootSuccess(), - testSuiteResultMessage.getResultPath(), - testSuiteResultMessage.getInfraLogPath(), - testSuiteResultMessage.getHostName(), - testSuiteResultMessage.getSuitePlan(), - testSuiteResultMessage.getSuiteVersion(), - testSuiteResultMessage.getSuiteName(), - testSuiteResultMessage.getSuiteBuildNumber(), - testSuiteResultMessage.getModulesDone(), - testSuiteResultMessage.getModulesTotal(), - testSuiteResultMessage.getBranch(), - testSuiteResultMessage.getTarget(), - testSuiteResultMessage.getBuildId(), - testSuiteResultMessage.getBuildSystemFingerprint(), - testSuiteResultMessage.getBuildVendorFingerprint(), - testSuiteResultMessage.getPassedTestCaseCount(), - testSuiteResultMessage.getFailedTestCaseCount()); - - testSuiteResultEntity.save(newTestSuiteFileEntity); - } catch (IOException e) { - ofy().delete().type(TestSuiteFileEntity.class).id(filePath).now(); - response.setStatus(HttpServletResponse.SC_BAD_REQUEST); - logger.log(Level.WARNING, "Invalid proto: " + e.getLocalizedMessage()); - } - } - } else { - logger.log(Level.INFO, "suite test found in datastore!"); - } - } - } -} |