diff options
author | Young Gyu Park <younggyu@google.com> | 2018-04-11 15:13:28 +0900 |
---|---|---|
committer | Young Gyu Park <younggyu@google.com> | 2018-04-11 15:13:28 +0900 |
commit | d3fe7f44e6d49e3a423b423c44b1625c771fe995 (patch) | |
tree | 01068f82863ba91753e9ff079a81e237573f11c4 /src | |
parent | 84be4f920deae58de6f74959947b5bf94f322401 (diff) | |
download | dashboard-d3fe7f44e6d49e3a423b423c44b1625c771fe995.tar.gz |
Reflecting the TestSuiteResultEntity change to TestDataForDevServletandroid-o-mr1-iot-preview-8o-mr1-iot-preview-8
Test: curl http://127.0.0.1:8080/api/test_data/suite -H "Content-Type: application/json"
Bug: 77884292
Change-Id: Ia0104ba6530d4e38c9cd695587960a82dce10693
Diffstat (limited to 'src')
-rw-r--r-- | src/main/java/com/android/vts/api/TestDataForDevServlet.java | 590 |
1 files changed, 366 insertions, 224 deletions
diff --git a/src/main/java/com/android/vts/api/TestDataForDevServlet.java b/src/main/java/com/android/vts/api/TestDataForDevServlet.java index 40a7e29..24f2671 100644 --- a/src/main/java/com/android/vts/api/TestDataForDevServlet.java +++ b/src/main/java/com/android/vts/api/TestDataForDevServlet.java @@ -51,6 +51,9 @@ import com.google.gson.GsonBuilder; import java.io.IOException; import java.io.InputStreamReader; import java.io.Reader; +import java.nio.file.FileSystems; +import java.nio.file.Path; +import java.nio.file.Paths; import java.time.Instant; import java.time.temporal.ChronoUnit; import java.util.Arrays; @@ -72,19 +75,23 @@ import javax.servlet.http.HttpServletResponse; /** Servlet for handling requests to add mock data in datastore. */ public class TestDataForDevServlet extends HttpServlet { - protected static final Logger logger = - Logger.getLogger(TestDataForDevServlet.class.getName()); + protected static final Logger logger = Logger.getLogger(TestDataForDevServlet.class.getName()); + + /** Google Cloud Storage project's default directory name for suite test result files */ + private static final String GCS_SUITE_TEST_FOLDER_NAME = + System.getProperty("GCS_SUITE_TEST_FOLDER_NAME"); /** datastore instance to save the test data into datastore through datastore library. */ private DatastoreService datastore = DatastoreServiceFactory.getDatastoreService(); - /** Gson is a Java library that can be used to convert Java Objects into their JSON representation. - * It can also be used to convert a JSON string to an equivalent Java object. + /** + * Gson is a Java library that can be used to convert Java Objects into their JSON + * representation. It can also be used to convert a JSON string to an equivalent Java object. */ private Gson gson = new GsonBuilder().create(); - /** TestReportData class for mapping test-report-data.json. - * This internal class's each fields will be automatically mapped - * to test-report-data.json file through Gson + /** + * TestReportData class for mapping test-report-data.json. This internal class's each fields + * will be automatically mapped to test-report-data.json file through Gson */ private class TestReportDataObject { private List<Test> testList; @@ -116,7 +123,6 @@ public class TestDataForDevServlet extends HttpServlet { private List<BuildTarget> buildTargetList; private List<Branch> branchList; - private class Coverage { private String group; private long coveredLineCount; @@ -159,7 +165,6 @@ public class TestDataForDevServlet extends HttpServlet { private class Branch { private String branchName; } - } } @@ -185,41 +190,102 @@ public class TestDataForDevServlet extends HttpServlet { } } - private Map<String, Object> generateSuiteTestData(HttpServletRequest request, HttpServletResponse response) { + private Map<String, Object> generateSuiteTestData( + HttpServletRequest request, HttpServletResponse response) { Map<String, Object> resultMap = new HashMap<>(); + String fileSeparator = FileSystems.getDefault().getSeparator(); Random rand = new Random(); List<String> branchList = Arrays.asList("master", "oc_mr", "oc"); - List<String> targetList = Arrays.asList("sailfish-userdebug", "marlin-userdebug", "taimen-userdebug", - "walleye-userdebug", "aosp_arm_a-userdebug"); - branchList.forEach(branch -> { - targetList.forEach(target -> { - IntStream.range(0, 10) - .forEach( - idx -> { - com.googlecode.objectify.Key<TestSuiteFileEntity> testSuiteFileParent = com.googlecode.objectify.Key.create(TestSuiteFileEntity.class, "suite_result/2019/04/06/132343.bin"); - TestSuiteResultEntity ttestSuiteResultEntity = - new TestSuiteResultEntity( - testSuiteFileParent, - Instant.now().minus(rand.nextInt(100), ChronoUnit.DAYS).getEpochSecond(), - Instant.now().minus(rand.nextInt(100), ChronoUnit.DAYS).getEpochSecond(), - "Suite Test Plan", - "Suite Version " + idx, - "Suite Build Number " + idx, - rand.nextInt(), - rand.nextInt(), - branch, - target, - Long.toString(Math.abs(rand.nextLong())), - "Build System Fingerprint " + idx, - "Build Vendor Fingerprint " + idx, - rand.nextInt(), - rand.nextInt() - ); - - ttestSuiteResultEntity.save(); - }); - }); - }); + List<String> targetList = + Arrays.asList( + "sailfish-userdebug", + "marlin-userdebug", + "taimen-userdebug", + "walleye-userdebug", + "aosp_arm_a-userdebug"); + branchList.forEach( + branch -> + targetList.forEach( + target -> + IntStream.range(0, 10) + .forEach( + idx -> { + String year = String.format("%04d", 2010 + idx); + String month = + String.format("%02d", rand.nextInt(12)); + String day = + String.format("%02d", rand.nextInt(30)); + String fileName = + String.format( + "%02d%02d%02d.bin", + rand.nextInt(23) + 1, + rand.nextInt(59) + 1, + rand.nextInt(59) + 1); + + List<String> pathList = + Arrays.asList( + GCS_SUITE_TEST_FOLDER_NAME == "" + ? "suite_result" + : GCS_SUITE_TEST_FOLDER_NAME, + year, + month, + day, + fileName); + + Path pathInfo = + Paths.get( + String.join( + fileSeparator, + pathList)); + + TestSuiteFileEntity newTestSuiteFileEntity = + new TestSuiteFileEntity( + pathInfo.toString()); + newTestSuiteFileEntity.save(); + + com.googlecode.objectify.Key< + TestSuiteFileEntity> + testSuiteFileParent = + com.googlecode.objectify.Key + .create( + TestSuiteFileEntity + .class, + newTestSuiteFileEntity + .getFilePath()); + TestSuiteResultEntity testSuiteResultEntity = + new TestSuiteResultEntity( + testSuiteFileParent, + Instant.now() + .minus( + rand.nextInt( + 100), + ChronoUnit.DAYS) + .getEpochSecond(), + Instant.now() + .minus( + rand.nextInt( + 100), + ChronoUnit.DAYS) + .getEpochSecond(), + "Suite Test Plan", + "Suite Version " + idx, + "Suite Build Number " + idx, + rand.nextInt(), + rand.nextInt(), + branch, + target, + Long.toString(Math.abs(rand.nextLong())), + "Build System Fingerprint " + + idx, + "Build Vendor Fingerprint " + + idx, + rand.nextInt(), + rand.nextInt()); + + testSuiteResultEntity.save(); + }) + ) + ); resultMap.put("result", "successfully generated!"); return resultMap; } @@ -268,197 +334,273 @@ public class TestDataForDevServlet extends HttpServlet { String testType = pathParts[1]; if (testType.equalsIgnoreCase("report")) { TestReportDataObject trdObj = - gson.fromJson(postJsonReader, TestReportDataObject.class); + gson.fromJson(postJsonReader, TestReportDataObject.class); logger.log(Level.INFO, "trdObj => " + trdObj); - trdObj.testList.forEach(test -> { - test.testRunList.forEach(testRun -> { - TestEntity testEntity = new TestEntity(testRun.testName); - - Key testRunKey = KeyFactory.createKey(testEntity.key, TestRunEntity.KIND, - testRun.startTimestamp); - - List<TestCaseReference> failingTestCases = new ArrayList<>(); - for (int idx = 0; idx < testRun.failingTestcaseIds.size(); idx++){ - failingTestCases.add( - new TestCaseReference(testRun.failingTestcaseIds.get(idx), - testRun.failingTestcaseOffsets.get(idx)) - ); - } - - TestStatusEntity testStatusEntity = new TestStatusEntity(testRun.testName, - testRun.startTimestamp, (int) testRun.passCount, failingTestCases.size(), - failingTestCases); - datastore.put(testStatusEntity.toEntity()); - - testRun.coverageList.forEach(testRunCoverage -> { - CoverageEntity coverageEntity = new CoverageEntity(testRunKey, - testRunCoverage.group, testRunCoverage.coveredLineCount, - testRunCoverage.totalLineCount, testRunCoverage.filePath, - testRunCoverage.projectName, testRunCoverage.projectVersion, - testRunCoverage.lineCoverage); - datastore.put(coverageEntity.toEntity()); - }); - - testRun.profilingList.forEach(testRunProfile -> { - ProfilingPointRunEntity profilingEntity = new ProfilingPointRunEntity( - testRunKey, testRunProfile.name, testRunProfile.type, - testRunProfile.regressionMode, testRunProfile.labels, - testRunProfile.values, testRunProfile.xLabel, testRunProfile.yLabel, - testRunProfile.options); - datastore.put(profilingEntity.toEntity()); + trdObj.testList.forEach( + test -> { + test.testRunList.forEach( + testRun -> { + TestEntity testEntity = new TestEntity(testRun.testName); + + Key testRunKey = + KeyFactory.createKey( + testEntity.key, + TestRunEntity.KIND, + testRun.startTimestamp); + + List<TestCaseReference> failingTestCases = + new ArrayList<>(); + for (int idx = 0; + idx < testRun.failingTestcaseIds.size(); + idx++) { + failingTestCases.add( + new TestCaseReference( + testRun.failingTestcaseIds.get(idx), + testRun.failingTestcaseOffsets.get( + idx))); + } + + TestStatusEntity testStatusEntity = + new TestStatusEntity( + testRun.testName, + testRun.startTimestamp, + (int) testRun.passCount, + failingTestCases.size(), + failingTestCases); + datastore.put(testStatusEntity.toEntity()); + + testRun.coverageList.forEach( + testRunCoverage -> { + CoverageEntity coverageEntity = + new CoverageEntity( + testRunKey, + testRunCoverage.group, + testRunCoverage + .coveredLineCount, + testRunCoverage.totalLineCount, + testRunCoverage.filePath, + testRunCoverage.projectName, + testRunCoverage.projectVersion, + testRunCoverage.lineCoverage); + datastore.put(coverageEntity.toEntity()); + }); + + testRun.profilingList.forEach( + testRunProfile -> { + ProfilingPointRunEntity profilingEntity = + new ProfilingPointRunEntity( + testRunKey, + testRunProfile.name, + testRunProfile.type, + testRunProfile.regressionMode, + testRunProfile.labels, + testRunProfile.values, + testRunProfile.xLabel, + testRunProfile.yLabel, + testRunProfile.options); + datastore.put(profilingEntity.toEntity()); + }); + + TestCaseRunEntity testCaseEntity = new TestCaseRunEntity(); + testRun.testCaseRunList.forEach( + testCaseRun -> { + for (int idx = 0; + idx < testCaseRun.testCaseNames.size(); + idx++) { + testCaseEntity.addTestCase( + testCaseRun.testCaseNames.get(idx), + testCaseRun.results.get(idx)); + } + }); + datastore.put(testCaseEntity.toEntity()); + + testRun.deviceInfoList.forEach( + deviceInfo -> { + DeviceInfoEntity deviceInfoEntity = + new DeviceInfoEntity( + testRunKey, + deviceInfo.branch, + deviceInfo.product, + deviceInfo.buildFlavor, + deviceInfo.buildId, + deviceInfo.abiBitness, + deviceInfo.abiName); + ; + datastore.put(deviceInfoEntity.toEntity()); + }); + + testRun.buildTargetList.forEach( + buildTarget -> { + BuildTargetEntity buildTargetEntity = + new BuildTargetEntity( + buildTarget.targetName); + datastore.put(buildTargetEntity.toEntity()); + }); + + testRun.branchList.forEach( + branch -> { + BranchEntity branchEntity = + new BranchEntity(branch.branchName); + datastore.put(branchEntity.toEntity()); + }); + + TestRunEntity testRunEntity = + new TestRunEntity( + testEntity.key, + TestRunType.fromNumber(testRun.type), + testRun.startTimestamp, + testRun.endTimestamp, + testRun.testBuildId, + testRun.hostName, + testRun.passCount, + testRun.failCount, + testRun.testCaseIds, + testRun.links, + testRun.coveredLineCount, + testRun.totalLineCount); + datastore.put(testRunEntity.toEntity()); + + Entity newTestEntity = testEntity.toEntity(); + + Transaction txn = datastore.beginTransaction(); + try { + // Check if test already exists in the datastore + try { + Entity oldTest = datastore.get(testEntity.key); + TestEntity oldTestEntity = + TestEntity.fromEntity(oldTest); + if (oldTestEntity == null + || !oldTestEntity.equals(testEntity)) { + datastore.put(newTestEntity); + } + } catch (EntityNotFoundException e) { + datastore.put(newTestEntity); + } + txn.commit(); + + } catch (ConcurrentModificationException + | DatastoreFailureException + | DatastoreTimeoutException e) { + logger.log( + Level.WARNING, + "Retrying test run insert: " + + newTestEntity.getKey()); + } finally { + if (txn.isActive()) { + logger.log( + Level.WARNING, + "Transaction rollback forced for run: " + + testRunEntity.key); + txn.rollback(); + } + } + }); }); - - TestCaseRunEntity testCaseEntity = new TestCaseRunEntity(); - testRun.testCaseRunList.forEach(testCaseRun -> { - for (int idx = 0; idx < testCaseRun.testCaseNames.size(); idx++){ - testCaseEntity.addTestCase(testCaseRun.testCaseNames.get(idx), - testCaseRun.results.get(idx)); + } else { + TestPlanReportDataObject tprdObj = + gson.fromJson(postJsonReader, TestPlanReportDataObject.class); + tprdObj.testPlanList.forEach( + testPlan -> { + Entity testPlanEntity = + new TestPlanEntity(testPlan.testPlanName).toEntity(); + List<Key> testRunKeys = new ArrayList<>(); + for (int idx = 0; idx < testPlan.testModules.size(); idx++) { + String test = testPlan.testModules.get(idx); + long time = testPlan.testTimes.get(idx); + Key parentKey = KeyFactory.createKey(TestEntity.KIND, test); + Key testRunKey = + KeyFactory.createKey(parentKey, TestRunEntity.KIND, time); + testRunKeys.add(testRunKey); } - }); - datastore.put(testCaseEntity.toEntity()); - - testRun.deviceInfoList.forEach(deviceInfo -> { - DeviceInfoEntity deviceInfoEntity = new DeviceInfoEntity( - testRunKey, deviceInfo.branch, deviceInfo.product, - deviceInfo.buildFlavor, deviceInfo.buildId, deviceInfo.abiBitness, - deviceInfo.abiName);; - datastore.put(deviceInfoEntity.toEntity()); - }); - - testRun.buildTargetList.forEach(buildTarget -> { - BuildTargetEntity buildTargetEntity = - new BuildTargetEntity(buildTarget.targetName); - datastore.put(buildTargetEntity.toEntity()); - }); - - testRun.branchList.forEach(branch -> { - BranchEntity branchEntity = new BranchEntity(branch.branchName); - datastore.put(branchEntity.toEntity()); - }); - - TestRunEntity testRunEntity = new TestRunEntity(testEntity.key, - TestRunType.fromNumber(testRun.type), testRun.startTimestamp, - testRun.endTimestamp, testRun.testBuildId, testRun.hostName, - testRun.passCount, testRun.failCount, testRun.testCaseIds, - testRun.links, testRun.coveredLineCount, testRun.totalLineCount); - datastore.put(testRunEntity.toEntity()); - - Entity newTestEntity = testEntity.toEntity(); - - Transaction txn = datastore.beginTransaction(); - try { - // Check if test already exists in the datastore - try { - Entity oldTest = datastore.get(testEntity.key); - TestEntity oldTestEntity = TestEntity.fromEntity(oldTest); - if (oldTestEntity == null || !oldTestEntity.equals(testEntity)) { - datastore.put(newTestEntity); + Map<Key, Entity> testRuns = datastore.get(testRunKeys); + long passCount = 0; + long failCount = 0; + long startTimestamp = -1; + long endTimestamp = -1; + String testBuildId = null; + TestRunType type = null; + Set<DeviceInfoEntity> devices = new HashSet<>(); + for (Key testRunKey : testRuns.keySet()) { + TestRunEntity testRun = + TestRunEntity.fromEntity(testRuns.get(testRunKey)); + if (testRun == null) { + continue; // not a valid test run + } + passCount += testRun.passCount; + failCount += testRun.failCount; + if (startTimestamp < 0 || testRunKey.getId() < startTimestamp) { + startTimestamp = testRunKey.getId(); + } + if (endTimestamp < 0 || testRun.endTimestamp > endTimestamp) { + endTimestamp = testRun.endTimestamp; + } + if (type == null) { + type = testRun.type; + } else if (type != testRun.type) { + type = TestRunType.OTHER; + } + testBuildId = testRun.testBuildId; + Query deviceInfoQuery = + new Query(DeviceInfoEntity.KIND).setAncestor(testRunKey); + for (Entity deviceInfoEntity : + datastore.prepare(deviceInfoQuery).asIterable()) { + DeviceInfoEntity device = + DeviceInfoEntity.fromEntity(deviceInfoEntity); + if (device == null) { + continue; // invalid entity + } + devices.add(device); } - } catch (EntityNotFoundException e) { - datastore.put(newTestEntity); } - txn.commit(); - - } catch (ConcurrentModificationException | DatastoreFailureException - | DatastoreTimeoutException e) { - logger.log(Level.WARNING, - "Retrying test run insert: " + newTestEntity.getKey()); - } finally { - if (txn.isActive()) { + if (startTimestamp < 0 || testBuildId == null || type == null) { logger.log( - Level.WARNING, - "Transaction rollback forced for run: " + testRunEntity.key); - txn.rollback(); + Level.WARNING, + "Couldn't infer test run information from runs."); + return; } - } - }); - }); - } else { - TestPlanReportDataObject tprdObj = - gson.fromJson(postJsonReader, TestPlanReportDataObject.class); - tprdObj.testPlanList.forEach(testPlan -> { - Entity testPlanEntity = new TestPlanEntity(testPlan.testPlanName).toEntity(); - List<Key> testRunKeys = new ArrayList<>(); - for (int idx = 0; idx < testPlan.testModules.size(); idx++) { - String test = testPlan.testModules.get(idx); - long time = testPlan.testTimes.get(idx); - Key parentKey = KeyFactory.createKey(TestEntity.KIND, test); - Key testRunKey = KeyFactory.createKey(parentKey, TestRunEntity.KIND, time); - testRunKeys.add(testRunKey); - } - Map<Key, Entity> testRuns = datastore.get(testRunKeys); - long passCount = 0; - long failCount = 0; - long startTimestamp = -1; - long endTimestamp = -1; - String testBuildId = null; - TestRunType type = null; - Set<DeviceInfoEntity> devices = new HashSet<>(); - for (Key testRunKey : testRuns.keySet()) { - TestRunEntity testRun = TestRunEntity.fromEntity(testRuns.get(testRunKey)); - if (testRun == null) { - continue; // not a valid test run - } - passCount += testRun.passCount; - failCount += testRun.failCount; - if (startTimestamp < 0 || testRunKey.getId() < startTimestamp) { - startTimestamp = testRunKey.getId(); - } - if (endTimestamp < 0 || testRun.endTimestamp > endTimestamp) { - endTimestamp = testRun.endTimestamp; - } - if (type == null) { - type = testRun.type; - } else if (type != testRun.type) { - type = TestRunType.OTHER; - } - testBuildId = testRun.testBuildId; - Query deviceInfoQuery = new Query(DeviceInfoEntity.KIND).setAncestor(testRunKey); - for (Entity deviceInfoEntity : datastore.prepare(deviceInfoQuery).asIterable()) { - DeviceInfoEntity device = DeviceInfoEntity.fromEntity(deviceInfoEntity); - if (device == null) { - continue; // invalid entity + TestPlanRunEntity testPlanRun = + new TestPlanRunEntity( + testPlanEntity.getKey(), + testPlan.testPlanName, + type, + startTimestamp, + endTimestamp, + testBuildId, + passCount, + failCount, + testRunKeys); + + // Create the device infos. + for (DeviceInfoEntity device : devices) { + datastore.put(device.copyWithParent(testPlanRun.key).toEntity()); } - devices.add(device); - } - } - if (startTimestamp < 0 || testBuildId == null || type == null) { - logger.log(Level.WARNING, "Couldn't infer test run information from runs."); - return; - } - TestPlanRunEntity testPlanRun = - new TestPlanRunEntity(testPlanEntity.getKey(), testPlan.testPlanName, - type, startTimestamp, endTimestamp, testBuildId, passCount, failCount, - testRunKeys); - - // Create the device infos. - for (DeviceInfoEntity device : devices) { - datastore.put(device.copyWithParent(testPlanRun.key).toEntity()); - } - datastore.put(testPlanRun.toEntity()); - - Transaction txn = datastore.beginTransaction(); - try { - // Check if test already exists in the database - try { - datastore.get(testPlanEntity.getKey()); - } catch (EntityNotFoundException e) { - datastore.put(testPlanEntity); - } - txn.commit(); - } catch (ConcurrentModificationException | DatastoreFailureException - | DatastoreTimeoutException e) { - logger.log(Level.WARNING, - "Retrying test plan insert: " + testPlanEntity.getKey()); - } finally { - if (txn.isActive()) { - logger.log( - Level.WARNING, - "Transaction rollback forced for plan run: " + testPlanRun.key); - txn.rollback(); - } - } - }); + datastore.put(testPlanRun.toEntity()); + + Transaction txn = datastore.beginTransaction(); + try { + // Check if test already exists in the database + try { + datastore.get(testPlanEntity.getKey()); + } catch (EntityNotFoundException e) { + datastore.put(testPlanEntity); + } + txn.commit(); + } catch (ConcurrentModificationException + | DatastoreFailureException + | DatastoreTimeoutException e) { + logger.log( + Level.WARNING, + "Retrying test plan insert: " + testPlanEntity.getKey()); + } finally { + if (txn.isActive()) { + logger.log( + Level.WARNING, + "Transaction rollback forced for plan run: " + + testPlanRun.key); + txn.rollback(); + } + } + }); } } else { logger.log(Level.WARNING, "URL path parameter is omitted!"); |