From 23a7cd3e808d8f950c0d7067f66f20a747c5427a Mon Sep 17 00:00:00 2001 From: Young Gyu Park Date: Thu, 29 Nov 2018 15:20:40 +0900 Subject: Fix a bug to upload a big entity exceeding 1MB. Test: go/vts-web-staging Bug: 119579271 Change-Id: Ic8d43035fbebe5adbd011662583cf98f43f84f41 --- .../com/android/vts/api/DatastoreRestServlet.java | 10 +--- .../android/vts/entity/ProfilingPointEntity.java | 1 + .../vts/entity/ProfilingPointRunEntity.java | 64 +++++++++++++++++++--- 3 files changed, 61 insertions(+), 14 deletions(-) (limited to 'src/main') diff --git a/src/main/java/com/android/vts/api/DatastoreRestServlet.java b/src/main/java/com/android/vts/api/DatastoreRestServlet.java index fb5ac65..fa0ae65 100644 --- a/src/main/java/com/android/vts/api/DatastoreRestServlet.java +++ b/src/main/java/com/android/vts/api/DatastoreRestServlet.java @@ -379,15 +379,13 @@ public class DatastoreRestServlet extends BaseApiServlet { List listEntity = (List) entity; if (listEntity.size() > 0 && listEntity.get(0) instanceof TestCaseRunEntity) { - List dashboardEntityList = - (List) entity; Map< com.googlecode.objectify.Key< TestCaseRunEntity>, TestCaseRunEntity> testCaseRunEntityMap = DashboardEntity.saveAll( - dashboardEntityList, + testCaseRunEntityList, this .MAX_ENTITY_SIZE_PER_TRANSACTION); @@ -401,11 +399,9 @@ public class DatastoreRestServlet extends BaseApiServlet { .collect(Collectors.toList()); } else if (listEntity.size() > 0 && listEntity.get(0) instanceof TestRunEntity) { - List dashboardEntityList = - (List) entity; - dashboardEntityList.get(0).setTestCaseIds(testCaseIds); + testRunEntityList.get(0).setTestCaseIds(testCaseIds); DashboardEntity.saveAll( - dashboardEntityList, + testRunEntityList, this.MAX_ENTITY_SIZE_PER_TRANSACTION); } else { List dashboardEntityList = diff --git a/src/main/java/com/android/vts/entity/ProfilingPointEntity.java b/src/main/java/com/android/vts/entity/ProfilingPointEntity.java index ac2994c..9e2cae6 100644 --- a/src/main/java/com/android/vts/entity/ProfilingPointEntity.java +++ b/src/main/java/com/android/vts/entity/ProfilingPointEntity.java @@ -109,6 +109,7 @@ public class ProfilingPointEntity implements DashboardEntity { this.regressionMode = regressionMode; this.xLabel = xLabel; this.yLabel = yLabel; + this.updated = new Date(); } /** diff --git a/src/main/java/com/android/vts/entity/ProfilingPointRunEntity.java b/src/main/java/com/android/vts/entity/ProfilingPointRunEntity.java index 48df727..124b61d 100644 --- a/src/main/java/com/android/vts/entity/ProfilingPointRunEntity.java +++ b/src/main/java/com/android/vts/entity/ProfilingPointRunEntity.java @@ -22,17 +22,20 @@ import com.android.vts.proto.VtsReportMessage.VtsProfilingType; import com.google.appengine.api.datastore.Entity; import com.google.appengine.api.datastore.Key; import com.google.appengine.api.datastore.KeyFactory; +import com.google.common.collect.Lists; import com.google.protobuf.ByteString; import com.googlecode.objectify.annotation.Cache; import com.googlecode.objectify.annotation.Id; import com.googlecode.objectify.annotation.Ignore; +import com.googlecode.objectify.annotation.Index; import com.googlecode.objectify.annotation.Parent; import java.util.ArrayList; +import java.util.Date; import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; +import java.util.Objects; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.extern.log4j.Log4j2; import static com.googlecode.objectify.ObjectifyService.ofy; @@ -40,10 +43,9 @@ import static com.googlecode.objectify.ObjectifyService.ofy; @Cache @Data @NoArgsConstructor +@Log4j2 /** Entity describing a profiling point execution. */ public class ProfilingPointRunEntity implements DashboardEntity { - protected static final Logger logger = - Logger.getLogger(ProfilingPointRunEntity.class.getName()); public static final String KIND = "ProfilingPointRun"; @@ -56,6 +58,9 @@ public class ProfilingPointRunEntity implements DashboardEntity { public static final String Y_LABEL = "yLabel"; public static final String OPTIONS = "options"; + /** This value will set the limit size of values array field */ + public static final int VALUE_SIZE_LIMIT = 50000; + @Ignore private Key key; @@ -89,6 +94,7 @@ public class ProfilingPointRunEntity implements DashboardEntity { private List options; /** When this record was created or updated */ + @Index Date updated; /** * Create a ProfilingPointRunEntity object. @@ -122,6 +128,7 @@ public class ProfilingPointRunEntity implements DashboardEntity { this.xLabel = xLabel; this.yLabel = yLabel; this.options = options; + this.updated = new Date(); } @@ -157,6 +164,7 @@ public class ProfilingPointRunEntity implements DashboardEntity { this.xLabel = xLabel; this.yLabel = yLabel; this.options = options; + this.updated = new Date(); } /** @@ -177,6 +185,46 @@ public class ProfilingPointRunEntity implements DashboardEntity { return VtsProfilingRegressionMode.forNumber(regressionMode); } + /** + * Save multi rows function when the record exceed the limit which is 1MB. + * + * @return ProfilingPointRunEntity's key value. + */ + public com.googlecode.objectify.Key saveMultiRow() { + if (this.getValues().size() > VALUE_SIZE_LIMIT) { + + List> partitionedValueList = + Lists.partition(this.getValues(), VALUE_SIZE_LIMIT); + int partitionedValueListSize = partitionedValueList.size(); + + List> partitionedLabelList = new ArrayList<>(); + if (Objects.nonNull(this.getLabels()) && this.getLabels().size() > VALUE_SIZE_LIMIT) { + partitionedLabelList = Lists.partition(this.getLabels(), VALUE_SIZE_LIMIT); + } + + com.googlecode.objectify.Key profilingPointRunEntityKey = null; + if (partitionedValueListSize < VALUE_SIZE_LIMIT) { + for (int index = 0; index < partitionedValueListSize; index++) { + if (index > 0) { + this.values.addAll(partitionedValueList.get(index)); + if (index < partitionedLabelList.size()) { + this.labels.addAll(partitionedLabelList.get(index)); + } + } else { + this.values = partitionedValueList.get(index); + if (index < partitionedLabelList.size()) { + this.labels = partitionedLabelList.get(index); + } + } + profilingPointRunEntityKey = ofy().save().entity(this).now(); + } + } + return profilingPointRunEntityKey; + } else { + return ofy().save().entity(this).now(); + } + } + /** Saving function for the instance of this class */ @Override public com.googlecode.objectify.Key save() { @@ -215,8 +263,7 @@ public class ProfilingPointRunEntity implements DashboardEntity { || !e.hasProperty(VALUES) || !e.hasProperty(X_LABEL) || !e.hasProperty(Y_LABEL)) { - logger.log( - Level.WARNING, "Missing profiling point attributes in entity: " + e.toString()); + log.error("Missing profiling point attributes in entity: " + e.toString()); return null; } try { @@ -239,7 +286,7 @@ public class ProfilingPointRunEntity implements DashboardEntity { parentKey, name, type, regressionMode, labels, values, xLabel, yLabel, options); } catch (ClassCastException exception) { // Invalid cast - logger.log(Level.WARNING, "Error parsing profiling point run entity.", exception); + log.warn("Error parsing profiling point run entity.", exception); } return null; } @@ -297,6 +344,9 @@ public class ProfilingPointRunEntity implements DashboardEntity { default: // should never happen return null; } + if (values.size() > VALUE_SIZE_LIMIT) { + values = values.subList(0, VALUE_SIZE_LIMIT); + } List options = null; if (profilingReport.getOptionsCount() > 0) { options = new ArrayList<>(); -- cgit v1.2.3