diff options
author | Young Gyu Park <younggyu@google.com> | 2018-04-05 16:13:15 +0900 |
---|---|---|
committer | Young Gyu Park <younggyu@google.com> | 2018-04-11 08:57:02 +0900 |
commit | 84be4f920deae58de6f74959947b5bf94f322401 (patch) | |
tree | 931d4b33c6130df5a77e7a485a75cf5483e165ab | |
parent | f64099ea656e8bf64acd85c179c6878510908b35 (diff) | |
download | dashboard-84be4f920deae58de6f74959947b5bf94f322401.tar.gz |
Basic structure setup for creating cron job servlet.
Test: mma
Bug: 77608122
Change-Id: Ica8a4594fb0e8f012e0700d76b7e42e27bbc8181
-rw-r--r-- | pom.xml | 1 | ||||
-rw-r--r-- | src/main/java/com/android/vts/api/TestDataForDevServlet.java | 13 | ||||
-rw-r--r-- | src/main/java/com/android/vts/api/TestSuiteResultRestServlet.java | 4 | ||||
-rw-r--r-- | src/main/java/com/android/vts/config/ObjectifyListener.java | 2 | ||||
-rw-r--r-- | src/main/java/com/android/vts/entity/TestSuiteFileEntity.java | 78 | ||||
-rw-r--r-- | src/main/java/com/android/vts/entity/TestSuiteResultEntity.java | 18 | ||||
-rw-r--r-- | src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java | 264 | ||||
-rw-r--r-- | src/main/java/com/android/vts/servlet/ShowGcsLogServlet.java | 24 | ||||
-rw-r--r-- | src/main/java/com/android/vts/util/GcsHelper.java | 42 | ||||
-rw-r--r-- | src/main/webapp/WEB-INF/appengine-web.xml | 1 | ||||
-rw-r--r-- | src/main/webapp/WEB-INF/cron.xml | 6 | ||||
-rw-r--r-- | src/main/webapp/WEB-INF/datastore-indexes.xml | 6 | ||||
-rw-r--r-- | src/main/webapp/WEB-INF/queue.xml | 6 | ||||
-rw-r--r-- | src/main/webapp/WEB-INF/web.xml | 10 |
14 files changed, 455 insertions, 20 deletions
@@ -39,6 +39,7 @@ <gcs.projectID></gcs.projectID> <gcs.keyFile></gcs.keyFile> <gcs.bucketName></gcs.bucketName> + <gcs.suiteTestFolderName></gcs.suiteTestFolderName> <maven.compiler.target>1.8</maven.compiler.target> <maven.compiler.source>1.8</maven.compiler.source> diff --git a/src/main/java/com/android/vts/api/TestDataForDevServlet.java b/src/main/java/com/android/vts/api/TestDataForDevServlet.java index 6a28b73..40a7e29 100644 --- a/src/main/java/com/android/vts/api/TestDataForDevServlet.java +++ b/src/main/java/com/android/vts/api/TestDataForDevServlet.java @@ -26,6 +26,7 @@ import com.android.vts.entity.TestEntity; import com.android.vts.entity.TestPlanEntity; import com.android.vts.entity.TestPlanRunEntity; import com.android.vts.entity.TestRunEntity; +import com.android.vts.entity.TestSuiteFileEntity; import com.android.vts.entity.TestSuiteResultEntity; import com.android.vts.entity.TestStatusEntity; @@ -52,7 +53,15 @@ import java.io.InputStreamReader; import java.io.Reader; import java.time.Instant; import java.time.temporal.ChronoUnit; -import java.util.*; +import java.util.Arrays; +import java.util.ArrayList; +import java.util.ConcurrentModificationException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.IntStream; @@ -187,8 +196,10 @@ public class TestDataForDevServlet extends HttpServlet { IntStream.range(0, 10) .forEach( idx -> { + com.googlecode.objectify.Key<TestSuiteFileEntity> testSuiteFileParent = com.googlecode.objectify.Key.create(TestSuiteFileEntity.class, "suite_result/2019/04/06/132343.bin"); TestSuiteResultEntity ttestSuiteResultEntity = new TestSuiteResultEntity( + testSuiteFileParent, Instant.now().minus(rand.nextInt(100), ChronoUnit.DAYS).getEpochSecond(), Instant.now().minus(rand.nextInt(100), ChronoUnit.DAYS).getEpochSecond(), "Suite Test Plan", diff --git a/src/main/java/com/android/vts/api/TestSuiteResultRestServlet.java b/src/main/java/com/android/vts/api/TestSuiteResultRestServlet.java index 3de5d06..4f53587 100644 --- a/src/main/java/com/android/vts/api/TestSuiteResultRestServlet.java +++ b/src/main/java/com/android/vts/api/TestSuiteResultRestServlet.java @@ -16,6 +16,7 @@ package com.android.vts.api; +import com.android.vts.entity.TestSuiteFileEntity; import com.android.vts.entity.TestSuiteResultEntity; import com.android.vts.proto.TestSuiteResultMessageProto.TestSuiteResultMessage; import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; @@ -24,6 +25,7 @@ import com.google.api.client.json.jackson.JacksonFactory; import com.google.api.services.oauth2.Oauth2; import com.google.api.services.oauth2.model.Tokeninfo; import com.google.gson.Gson; +import com.googlecode.objectify.Key; import org.apache.commons.codec.binary.Base64; import javax.servlet.http.HttpServlet; @@ -73,8 +75,10 @@ public class TestSuiteResultRestServlet extends HttpServlet { Tokeninfo tokenInfo = oauth2.tokeninfo().setAccessToken(accessToken).execute(); if (tokenInfo.getIssuedTo().equals(SERVICE_CLIENT_ID)) { + Key<TestSuiteFileEntity> testSuiteFileParent = Key.create(TestSuiteFileEntity.class, "suite_result/2019/04/06/132343.bin"); TestSuiteResultEntity testSuiteResultEntity = new TestSuiteResultEntity( + testSuiteFileParent, testSuiteResultMessage.getStartTime(), testSuiteResultMessage.getEndTime(), testSuiteResultMessage.getSuitePlan(), diff --git a/src/main/java/com/android/vts/config/ObjectifyListener.java b/src/main/java/com/android/vts/config/ObjectifyListener.java index cee9dd1..61ca159 100644 --- a/src/main/java/com/android/vts/config/ObjectifyListener.java +++ b/src/main/java/com/android/vts/config/ObjectifyListener.java @@ -16,6 +16,7 @@ package com.android.vts.config; +import com.android.vts.entity.TestSuiteFileEntity; import com.android.vts.entity.TestSuiteResultEntity; import com.googlecode.objectify.ObjectifyFactory; import com.googlecode.objectify.ObjectifyService; @@ -43,6 +44,7 @@ public class ObjectifyListener implements ServletContextListener { @Override public void contextInitialized(ServletContextEvent servletContextEvent) { ObjectifyFactory objectifyFactory = ObjectifyService.factory(); + objectifyFactory.register(TestSuiteFileEntity.class); objectifyFactory.register(TestSuiteResultEntity.class); objectifyFactory.begin(); logger.log(Level.INFO, "Value Initialized from context."); diff --git a/src/main/java/com/android/vts/entity/TestSuiteFileEntity.java b/src/main/java/com/android/vts/entity/TestSuiteFileEntity.java new file mode 100644 index 0000000..2331e42 --- /dev/null +++ b/src/main/java/com/android/vts/entity/TestSuiteFileEntity.java @@ -0,0 +1,78 @@ +/* + * Copyright (C) 2018 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.android.vts.entity; + +import com.googlecode.objectify.annotation.Cache; +import com.googlecode.objectify.annotation.Entity; +import com.googlecode.objectify.annotation.Id; +import com.googlecode.objectify.annotation.Index; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Date; +import java.util.List; + +import static com.googlecode.objectify.ObjectifyService.ofy; + +/** Entity Class for Suite Test File Info */ +@Cache +@Entity +@EqualsAndHashCode(of = "id") +@NoArgsConstructor +public class TestSuiteFileEntity { + + /** Test Suite full file path field */ + @Id @Getter @Setter String filePath; + + /** Test Suite year field in the filePath */ + @Index @Getter @Setter int year; + + /** Test Suite month field in the filePath */ + @Index @Getter @Setter int month; + + /** Test Suite day field in the filePath */ + @Index @Getter @Setter int day; + + /** Test Suite file name field */ + @Index @Getter @Setter String fileName; + + /** When this record was created or updated */ + @Index @Getter Date updated; + + /** Construction function for TestSuiteResultEntity Class */ + public TestSuiteFileEntity(String filePath) { + this.filePath = filePath; + Path pathInfo = Paths.get(filePath); + if (pathInfo.getNameCount() > 3) { + this.year = Integer.valueOf(pathInfo.getName(1).toString()); + this.month = Integer.valueOf(pathInfo.getName(2).toString()); + this.day = Integer.valueOf(pathInfo.getName(3).toString()); + this.fileName = pathInfo.getFileName().toString(); + } + } + + /** Saving function for the instance of this class */ + public void save() { + this.updated = new Date(); + ofy().save().entity(this).now(); + } +} diff --git a/src/main/java/com/android/vts/entity/TestSuiteResultEntity.java b/src/main/java/com/android/vts/entity/TestSuiteResultEntity.java index 8d7fe87..b5625ca 100644 --- a/src/main/java/com/android/vts/entity/TestSuiteResultEntity.java +++ b/src/main/java/com/android/vts/entity/TestSuiteResultEntity.java @@ -16,10 +16,12 @@ package com.android.vts.entity; +import com.googlecode.objectify.Key; import com.googlecode.objectify.annotation.Cache; import com.googlecode.objectify.annotation.Entity; import com.googlecode.objectify.annotation.Id; import com.googlecode.objectify.annotation.Index; +import com.googlecode.objectify.annotation.Parent; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NoArgsConstructor; @@ -37,6 +39,9 @@ import static com.googlecode.objectify.ObjectifyService.ofy; @NoArgsConstructor public class TestSuiteResultEntity { + @Parent + Key<TestSuiteFileEntity> testSuiteFileEntityKey; + /** Test Suite start time field */ @Id @Getter @Setter Long startTime; @@ -87,6 +92,7 @@ public class TestSuiteResultEntity { /** Construction function for TestSuiteResultEntity Class */ public TestSuiteResultEntity( + Key<TestSuiteFileEntity> testSuiteFileEntityKey, Long startTime, Long endTime, String suitePlan, @@ -101,6 +107,7 @@ public class TestSuiteResultEntity { String buildVendorFingerprint, int passedTestCaseCount, int failedTestCaseCount) { + this.testSuiteFileEntityKey = testSuiteFileEntityKey; this.startTime = startTime; this.endTime = endTime; this.suitePlan = suitePlan; @@ -115,14 +122,19 @@ public class TestSuiteResultEntity { this.buildVendorFingerprint = buildVendorFingerprint; this.passedTestCaseCount = passedTestCaseCount; this.failedTestCaseCount = failedTestCaseCount; - this.passedTestCaseRatio = - passedTestCaseCount / (passedTestCaseCount + failedTestCaseCount) * 100; + + int totalTestCaseCount = passedTestCaseCount + failedTestCaseCount; + if ( totalTestCaseCount <= 0 ) { + this.passedTestCaseRatio = 0; + } else { + this.passedTestCaseRatio = passedTestCaseCount / totalTestCaseCount * 100; + } } /** Saving function for the instance of this class */ public void save() { this.updated = new Date(); - ofy().defer().save().entity(this); + ofy().save().entity(this).now(); } public List<? extends TestSuiteResultEntity> getTestSuitePlans() { diff --git a/src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java b/src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java new file mode 100644 index 0000000..60f5645 --- /dev/null +++ b/src/main/java/com/android/vts/job/VtsSuiteTestJobServlet.java @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2018 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you + * may not use this file except in compliance with the License. You may + * obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + * implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package com.android.vts.job; + +import com.android.vts.entity.TestEntity; +import com.android.vts.entity.TestRunEntity; +import com.android.vts.entity.TestStatusEntity; +import com.android.vts.entity.TestSuiteFileEntity; +import com.android.vts.entity.TestSuiteResultEntity; +import com.android.vts.proto.TestSuiteResultMessageProto; +import com.android.vts.util.EmailHelper; +import com.android.vts.util.FilterUtil; +import com.android.vts.util.GcsHelper; +import com.android.vts.util.TaskQueueHelper; +import com.android.vts.util.TimeUtil; +import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; +import com.google.api.client.http.javanet.NetHttpTransport; +import com.google.api.client.json.jackson.JacksonFactory; +import com.google.api.services.oauth2.Oauth2; +import com.google.api.services.oauth2.model.Tokeninfo; +import com.google.appengine.api.datastore.DatastoreService; +import com.google.appengine.api.datastore.DatastoreServiceFactory; +import com.google.appengine.api.datastore.Query.Filter; +import com.google.appengine.api.datastore.Query.SortDirection; +import com.google.appengine.api.memcache.ErrorHandlers; +import com.google.appengine.api.memcache.MemcacheService; +import com.google.appengine.api.memcache.MemcacheServiceFactory; +import com.google.appengine.api.taskqueue.Queue; +import com.google.appengine.api.taskqueue.QueueFactory; +import com.google.appengine.api.taskqueue.TaskOptions; +import com.google.cloud.storage.Blob; +import com.google.cloud.storage.Bucket; +import com.google.cloud.storage.Storage; +import com.googlecode.objectify.Key; +import com.googlecode.objectify.Result; +import org.apache.commons.codec.binary.Base64; + +import javax.mail.Message; +import javax.mail.MessagingException; +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.UnsupportedEncodingException; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.concurrent.TimeUnit; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import static com.googlecode.objectify.ObjectifyService.ofy; + +/** Suite Test result file processing job. */ +public class VtsSuiteTestJobServlet extends HttpServlet { + + private static final String SUITE_TEST_URL = "/cron/test_suite_report_gcs_monitor"; + + private static final String SERVICE_CLIENT_ID = System.getProperty("SERVICE_CLIENT_ID"); + private static final String SERVICE_NAME = "VTS Dashboard"; + + private final Logger logger = Logger.getLogger(this.getClass().getName()); + + /** Google Cloud Storage project's key file to access the storage */ + private static final String GCS_KEY_FILE = System.getProperty("GCS_KEY_FILE"); + /** Google Cloud Storage project's default bucket name for vtslab log files */ + private static final String GCS_BUCKET_NAME = System.getProperty("GCS_BUCKET_NAME"); + /** Google Cloud Storage project's default directory name for suite test result files */ + private static final String GCS_SUITE_TEST_FOLDER_NAME = + System.getProperty("GCS_SUITE_TEST_FOLDER_NAME"); + + public static final String QUEUE = "suiteTestQueue"; + + /** + * This is the key file to access vtslab-gcs project. It will allow the dashboard to have a full + * control of the bucket. + */ + private InputStream keyFileInputStream; + + /** This is the instance of java google storage library */ + private Storage storage; + + /** This is the instance of App Engine memcache service java library */ + private MemcacheService syncCache = MemcacheServiceFactory.getMemcacheService(); + + private final DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); + + @Override + public void init(ServletConfig servletConfig) throws ServletException { + super.init(servletConfig); + + this.keyFileInputStream = + this.getServletContext().getResourceAsStream("/WEB-INF/keys/" + GCS_KEY_FILE); + + Optional<Storage> optionalStorage = GcsHelper.getStorage(this.keyFileInputStream); + if (optionalStorage.isPresent()) { + this.storage = optionalStorage.get(); + } else { + logger.log(Level.SEVERE, "Error on getting storage instance!"); + throw new ServletException("Creating storage instance exception!"); + } + syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO)); + } + + @Override + public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException { + + logger.log(Level.INFO, "Job Started!!!!!!!!!!!!!"); + + Queue queue = QueueFactory.getQueue(QUEUE); + + List<TaskOptions> tasks = new ArrayList<>(); + + String fileSeparator = FileSystems.getDefault().getSeparator(); + Date today = new Date(); + String year = new SimpleDateFormat("yyyy").format(today); + String month = new SimpleDateFormat("MM").format(today); + String day = new SimpleDateFormat("dd").format(today); + + List<String> pathList = Arrays.asList(GCS_SUITE_TEST_FOLDER_NAME, year, month, day); + Path pathInfo = Paths.get(String.join(fileSeparator, pathList)); + + List<TestSuiteFileEntity> testSuiteFileEntityList = + ofy().load() + .type(TestSuiteFileEntity.class) + .filter("year", Integer.parseInt(year)) + .filter("month", Integer.parseInt(month)) + .filter("day", Integer.parseInt(day)) + .list(); + + List<String> filePathList = + testSuiteFileEntityList + .stream() + .map(testSuiteFile -> testSuiteFile.getFilePath()) + .collect(Collectors.toList()); + + Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME); + + Storage.BlobListOption[] listOptions = + new Storage.BlobListOption[] { + Storage.BlobListOption.prefix(pathInfo.toString() + fileSeparator) + }; + + Iterable<Blob> blobIterable = vtsReportBucket.list(listOptions).iterateAll(); + Iterator<Blob> blobIterator = blobIterable.iterator(); + while (blobIterator.hasNext()) { + Blob blob = blobIterator.next(); + if (blob.isDirectory()) { + logger.log(Level.INFO, blob.getName() + " directory will be skipped!"); + } else { + if (filePathList.contains(blob.getName())) { + logger.log(Level.INFO, "filePathList contain => " + blob.getName()); + } else if (blob.getName().endsWith(fileSeparator)) { + logger.log(Level.INFO, blob.getName() + " endswith slash!"); + } else { + TaskOptions task = + TaskOptions.Builder.withUrl(SUITE_TEST_URL) + .param("filePath", blob.getName()) + .method(TaskOptions.Method.POST); + tasks.add(task); + } + } + } + TaskQueueHelper.addToQueue(queue, tasks); + } + + @Override + public void doPost(HttpServletRequest request, HttpServletResponse response) + throws IOException { + + String filePath = request.getParameter("filePath"); + if (Objects.isNull(filePath)) { + logger.log(Level.WARNING, "filePath parameter is null!"); + } else { + logger.log(Level.INFO, "filePath parameter => " + filePath); + + Key<TestSuiteFileEntity> testSuiteFileEntityKey = + Key.create(TestSuiteFileEntity.class, filePath); + TestSuiteFileEntity testSuiteFileEntity = + ofy().load() + .type(TestSuiteFileEntity.class) + .filterKey(testSuiteFileEntityKey) + .first() + .now(); + + if (Objects.isNull(testSuiteFileEntity)) { + Blob blobFile = (Blob) this.syncCache.get(filePath); + if (Objects.isNull(blobFile)) { + Bucket vtsReportBucket = this.storage.get(GCS_BUCKET_NAME); + blobFile = vtsReportBucket.get(filePath); + this.syncCache.put(filePath, blobFile); + } + + if (blobFile.exists()) { + TestSuiteFileEntity newTestSuiteFileEntity = new TestSuiteFileEntity(filePath); + newTestSuiteFileEntity.save(); + TestSuiteResultMessageProto.TestSuiteResultMessage testSuiteResultMessage; + try { + // String payload = blobFile.getContent().collect(Collectors.joining()); + // byte[] value = Base64.decodeBase64(payload); + testSuiteResultMessage = + TestSuiteResultMessageProto.TestSuiteResultMessage.parseFrom( + blobFile.getContent()); + + TestSuiteResultEntity testSuiteResultEntity = + new TestSuiteResultEntity( + testSuiteFileEntityKey, + testSuiteResultMessage.getStartTime(), + testSuiteResultMessage.getEndTime(), + testSuiteResultMessage.getSuitePlan(), + testSuiteResultMessage.getSuiteVersion(), + testSuiteResultMessage.getSuiteBuildNumber(), + testSuiteResultMessage.getModulesDone(), + testSuiteResultMessage.getModulesTotal(), + testSuiteResultMessage.getBranch(), + testSuiteResultMessage.getTarget(), + testSuiteResultMessage.getBuildId(), + testSuiteResultMessage.getBuildSystemFingerprint(), + testSuiteResultMessage.getBuildVendorFingerprint(), + testSuiteResultMessage.getPassedTestCaseCount(), + testSuiteResultMessage.getFailedTestCaseCount()); + + testSuiteResultEntity.save(); + } catch (IOException e) { + ofy().delete().type(TestSuiteFileEntity.class).id(filePath).now(); + response.setStatus(HttpServletResponse.SC_BAD_REQUEST); + logger.log(Level.WARNING, "Invalid proto: " + e.getLocalizedMessage()); + } + } + } else { + logger.log(Level.INFO, "suite test found in datastore!"); + } + } + } +} diff --git a/src/main/java/com/android/vts/servlet/ShowGcsLogServlet.java b/src/main/java/com/android/vts/servlet/ShowGcsLogServlet.java index abde1f4..82ed161 100644 --- a/src/main/java/com/android/vts/servlet/ShowGcsLogServlet.java +++ b/src/main/java/com/android/vts/servlet/ShowGcsLogServlet.java @@ -16,6 +16,7 @@ package com.android.vts.servlet; +import com.android.vts.util.GcsHelper; import com.google.appengine.api.memcache.ErrorHandlers; import com.google.appengine.api.memcache.MemcacheService; import com.google.appengine.api.memcache.MemcacheServiceFactory; @@ -53,8 +54,6 @@ public class ShowGcsLogServlet extends BaseServlet { private static final String GCS_LOG_JSP = "WEB-INF/jsp/show_gcs_log.jsp"; - /** Google Cloud Storage project ID */ - private static final String GCS_PROJECT_ID = System.getProperty("GCS_PROJECT_ID"); /** Google Cloud Storage project's key file to access the storage */ private static final String GCS_KEY_FILE = System.getProperty("GCS_KEY_FILE"); /** Google Cloud Storage project's default bucket name for vtslab log files */ @@ -75,23 +74,16 @@ public class ShowGcsLogServlet extends BaseServlet { @Override public void init(ServletConfig cfg) throws ServletException { super.init(cfg); - keyFileInputStream = + + this.keyFileInputStream = this.getServletContext().getResourceAsStream("/WEB-INF/keys/" + GCS_KEY_FILE); - if (keyFileInputStream == null) { - logger.log(Level.SEVERE, "Error GCS key file is not exiting. Check key file!"); + Optional<Storage> optionalStorage = GcsHelper.getStorage(this.keyFileInputStream); + if (optionalStorage.isPresent()) { + this.storage = optionalStorage.get(); } else { - try { - storage = - StorageOptions.newBuilder() - .setProjectId(GCS_PROJECT_ID) - .setCredentials( - ServiceAccountCredentials.fromStream(keyFileInputStream)) - .build() - .getService(); - } catch (IOException e) { - logger.log(Level.SEVERE, "Error on creating storage instance!"); - } + logger.log(Level.SEVERE, "Error on getting storage instance!"); + throw new ServletException("Creating storage instance exception!"); } syncCache.setErrorHandler(ErrorHandlers.getConsistentLogAndContinue(Level.INFO)); } diff --git a/src/main/java/com/android/vts/util/GcsHelper.java b/src/main/java/com/android/vts/util/GcsHelper.java new file mode 100644 index 0000000..bb2b5be --- /dev/null +++ b/src/main/java/com/android/vts/util/GcsHelper.java @@ -0,0 +1,42 @@ +package com.android.vts.util; + +import com.google.auth.oauth2.ServiceAccountCredentials; +import com.google.cloud.storage.Storage; +import com.google.cloud.storage.StorageOptions; + +import javax.servlet.ServletContext; +import java.io.IOException; +import java.io.InputStream; +import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** GcsHelper, a helper class for interacting with Google Cloud Storage. */ +public class GcsHelper { + private static final Logger logger = Logger.getLogger(GcsHelper.class.getName()); + + /** Google Cloud Storage project ID */ + private static final String GCS_PROJECT_ID = System.getProperty("GCS_PROJECT_ID"); + + public static Optional<Storage> getStorage(InputStream keyFileInputStream) { + + if (keyFileInputStream == null) { + logger.log(Level.SEVERE, "Error GCS key file is not exiting. Check key file!"); + return Optional.empty(); + } else { + try { + Storage storage = + StorageOptions.newBuilder() + .setProjectId(GCS_PROJECT_ID) + .setCredentials( + ServiceAccountCredentials.fromStream(keyFileInputStream)) + .build() + .getService(); + return Optional.of(storage); + } catch (IOException e) { + logger.log(Level.SEVERE, "Error on creating storage instance!"); + return Optional.empty(); + } + } + } +} diff --git a/src/main/webapp/WEB-INF/appengine-web.xml b/src/main/webapp/WEB-INF/appengine-web.xml index fed561d..bc527c0 100644 --- a/src/main/webapp/WEB-INF/appengine-web.xml +++ b/src/main/webapp/WEB-INF/appengine-web.xml @@ -30,6 +30,7 @@ <property name="GCS_PROJECT_ID" value="${gcs.projectID}" /> <property name="GCS_KEY_FILE" value="${gcs.keyFile}" /> <property name="GCS_BUCKET_NAME" value="${gcs.bucketName}" /> + <property name="GCS_SUITE_TEST_FOLDER_NAME" value="${gcs.suiteTestFolderName}" /> </system-properties> </appengine-web-app>
\ No newline at end of file diff --git a/src/main/webapp/WEB-INF/cron.xml b/src/main/webapp/WEB-INF/cron.xml index bdd9eaa..2dc72f8 100644 --- a/src/main/webapp/WEB-INF/cron.xml +++ b/src/main/webapp/WEB-INF/cron.xml @@ -27,4 +27,10 @@ Copyright 2016 Google Inc. All Rights Reserved. <schedule>every day 07:30</schedule> <timezone>America/Los_Angeles</timezone> </cron> + <cron> + <url>/cron/test_suite_report_gcs_monitor</url> + <description>Process suite test report from gcs.</description> + <schedule>every 5 mins</schedule> + <timezone>America/Los_Angeles</timezone> + </cron> </cronentries>
\ No newline at end of file diff --git a/src/main/webapp/WEB-INF/datastore-indexes.xml b/src/main/webapp/WEB-INF/datastore-indexes.xml index 1b96cff..244ad38 100644 --- a/src/main/webapp/WEB-INF/datastore-indexes.xml +++ b/src/main/webapp/WEB-INF/datastore-indexes.xml @@ -101,6 +101,12 @@ <property name="buildFlavor" direction="asc"/> </datastore-index> + <datastore-index kind="TestSuiteFileEntity" ancestor="false" source="manual"> + <property name="day" direction="asc"/> + <property name="month" direction="asc"/> + <property name="year" direction="asc"/> + </datastore-index> + <datastore-index kind="TestSuiteResultEntity" ancestor="false" source="manual"> <property name="suitePlan" direction="asc"/> <property name="branch" direction="asc"/> diff --git a/src/main/webapp/WEB-INF/queue.xml b/src/main/webapp/WEB-INF/queue.xml index 0fccfeb..8900015 100644 --- a/src/main/webapp/WEB-INF/queue.xml +++ b/src/main/webapp/WEB-INF/queue.xml @@ -20,4 +20,10 @@ Copyright (C) 2017 The Android Open Source Project <rate>200/s</rate> <bucket-size>40</bucket-size> </queue> + <queue> + <name>suiteTestQueue</name> + <rate>20/s</rate> + <bucket-size>40</bucket-size> + <max-concurrent-requests>10</max-concurrent-requests> + </queue> </queue-entries>
\ No newline at end of file diff --git a/src/main/webapp/WEB-INF/web.xml b/src/main/webapp/WEB-INF/web.xml index ad2159d..f5ff208 100644 --- a/src/main/webapp/WEB-INF/web.xml +++ b/src/main/webapp/WEB-INF/web.xml @@ -165,6 +165,11 @@ Copyright 2016 Google Inc. All Rights Reserved. <servlet-class>com.android.vts.job.VtsInactivityJobServlet</servlet-class> </servlet> +<servlet> + <servlet-name>suite_test_report_gcs_monitor_job</servlet-name> + <servlet-class>com.android.vts.job.VtsSuiteTestJobServlet</servlet-class> +</servlet> + <servlet-mapping> <servlet-name>dashboard_main</servlet-name> <url-pattern>/</url-pattern> @@ -300,6 +305,11 @@ Copyright 2016 Google Inc. All Rights Reserved. <url-pattern>/cron/vts_inactivity_job/*</url-pattern> </servlet-mapping> +<servlet-mapping> + <servlet-name>suite_test_report_gcs_monitor_job</servlet-name> + <url-pattern>/cron/test_suite_report_gcs_monitor/*</url-pattern> +</servlet-mapping> + <security-constraint> <web-resource-collection> <web-resource-name>tasks</web-resource-name> |