1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
|
/**
* Copyright 2016 Google Inc. All Rights Reserved.
*
* <p>Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
* except in compliance with the License. You may obtain a copy of the License at
*
* <p>http://www.apache.org/licenses/LICENSE-2.0
*
* <p>Unless required by applicable law or agreed to in writing, software distributed under the
* License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.android.vts.util;
import com.android.vts.entity.ApiCoverageEntity;
import com.android.vts.entity.BranchEntity;
import com.android.vts.entity.BuildTargetEntity;
import com.android.vts.entity.CodeCoverageEntity;
import com.android.vts.entity.CoverageEntity;
import com.android.vts.entity.DeviceInfoEntity;
import com.android.vts.entity.ProfilingPointRunEntity;
import com.android.vts.entity.TestCaseRunEntity;
import com.android.vts.entity.TestEntity;
import com.android.vts.entity.TestPlanEntity;
import com.android.vts.entity.TestPlanRunEntity;
import com.android.vts.entity.TestRunEntity;
import com.android.vts.entity.TestRunEntity.TestRunType;
import com.android.vts.job.VtsAlertJobServlet;
import com.android.vts.job.VtsCoverageAlertJobServlet;
import com.android.vts.job.VtsProfilingStatsJobServlet;
import com.android.vts.proto.VtsReportMessage.AndroidDeviceInfoMessage;
import com.android.vts.proto.VtsReportMessage.ApiCoverageReportMessage;
import com.android.vts.proto.VtsReportMessage.CoverageReportMessage;
import com.android.vts.proto.VtsReportMessage.HalInterfaceMessage;
import com.android.vts.proto.VtsReportMessage.LogMessage;
import com.android.vts.proto.VtsReportMessage.ProfilingReportMessage;
import com.android.vts.proto.VtsReportMessage.TestCaseReportMessage;
import com.android.vts.proto.VtsReportMessage.TestCaseResult;
import com.android.vts.proto.VtsReportMessage.TestPlanReportMessage;
import com.android.vts.proto.VtsReportMessage.TestReportMessage;
import com.android.vts.proto.VtsReportMessage.UrlResourceMessage;
import com.google.appengine.api.datastore.DatastoreFailureException;
import com.google.appengine.api.datastore.DatastoreService;
import com.google.appengine.api.datastore.DatastoreServiceFactory;
import com.google.appengine.api.datastore.DatastoreTimeoutException;
import com.google.appengine.api.datastore.Entity;
import com.google.appengine.api.datastore.EntityNotFoundException;
import com.google.appengine.api.datastore.FetchOptions;
import com.google.appengine.api.datastore.Key;
import com.google.appengine.api.datastore.KeyFactory;
import com.google.appengine.api.datastore.Query;
import com.google.appengine.api.datastore.Query.Filter;
import com.google.appengine.api.datastore.Query.FilterOperator;
import com.google.appengine.api.datastore.Query.FilterPredicate;
import com.google.appengine.api.datastore.Transaction;
import com.google.appengine.api.datastore.TransactionOptions;
import com.google.common.collect.Lists;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.ConcurrentModificationException;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
import java.util.stream.Collectors;
/**
* DatastoreHelper, a helper class for interacting with Cloud Datastore.
*/
public class DatastoreHelper {
/**
* The default kind name for datastore
*/
public static final String NULL_ENTITY_KIND = "nullEntity";
public static final int MAX_WRITE_RETRIES = 5;
/**
* This variable is for maximum number of entities per transaction You can find the detail here
* (https://cloud.google.com/datastore/docs/concepts/limits)
*/
public static final int MAX_ENTITY_SIZE_PER_TRANSACTION = 300;
protected static final Logger logger = Logger.getLogger(DatastoreHelper.class.getName());
private static final DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
/**
* Get query fetch options for large batches of entities.
*
* @return FetchOptions with a large chunk and prefetch size.
*/
public static FetchOptions getLargeBatchOptions() {
return FetchOptions.Builder.withChunkSize(1000).prefetchSize(1000);
}
/**
* Returns true if there are data points newer than lowerBound in the results table.
*
* @param parentKey The parent key to use in the query.
* @param kind The query entity kind.
* @param lowerBound The (exclusive) lower time bound, long, microseconds.
* @return boolean True if there are newer data points.
*/
public static boolean hasNewer(Key parentKey, String kind, Long lowerBound) {
if (lowerBound == null || lowerBound <= 0) {
return false;
}
DatastoreService datastore = DatastoreServiceFactory.getDatastoreService();
Key startKey = KeyFactory.createKey(parentKey, kind, lowerBound);
Filter startFilter =
new FilterPredicate(
Entity.KEY_RESERVED_PROPERTY, FilterOperator.GREATER_THAN, startKey);
Query q = new Query(kind).setAncestor(parentKey).setFilter(startFilter).setKeysOnly();
return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0;
}
/**
* Returns true if there are data points older than upperBound in the table.
*
* @param parentKey The parent key to use in the query.
* @param kind The query entity kind.
* @param upperBound The (exclusive) upper time bound, long, microseconds.
* @return boolean True if there are older data points.
*/
public static boolean hasOlder(Key parentKey, String kind, Long upperBound) {
if (upperBound == null || upperBound <= 0) {
return false;
}
Key endKey = KeyFactory.createKey(parentKey, kind, upperBound);
Filter endFilter =
new FilterPredicate(Entity.KEY_RESERVED_PROPERTY, FilterOperator.LESS_THAN, endKey);
Query q = new Query(kind).setAncestor(parentKey).setFilter(endFilter).setKeysOnly();
return datastore.prepare(q).countEntities(FetchOptions.Builder.withLimit(1)) > 0;
}
/**
* Get all of the devices branches.
*
* @return a list of all branches.
*/
public static List<String> getAllBranches() {
Query query = new Query(BranchEntity.KIND).setKeysOnly();
List<String> branches = new ArrayList<>();
for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) {
branches.add(e.getKey().getName());
}
return branches;
}
/**
* Get all of the device build flavors.
*
* @return a list of all device build flavors.
*/
public static List<String> getAllBuildFlavors() {
Query query = new Query(BuildTargetEntity.KIND).setKeysOnly();
List<String> devices = new ArrayList<>();
for (Entity e : datastore.prepare(query).asIterable(getLargeBatchOptions())) {
devices.add(e.getKey().getName());
}
return devices;
}
/**
* Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times and withXG of
* false value
*
* @param entity The entity that you want to insert to datastore.
* @param entityList The list of entity for using datastore put method.
*/
private static boolean datastoreTransactionalRetry(Entity entity, List<Entity> entityList) {
return datastoreTransactionalRetryWithXG(entity, entityList, false);
}
/**
* Datastore Transactional process for data insertion with MAX_WRITE_RETRIES times
*
* @param entity The entity that you want to insert to datastore.
* @param entityList The list of entity for using datastore put method.
*/
private static boolean datastoreTransactionalRetryWithXG(
Entity entity, List<Entity> entityList, boolean withXG) {
int retries = 0;
while (true) {
Transaction txn;
if (withXG) {
TransactionOptions options = TransactionOptions.Builder.withXG(withXG);
txn = datastore.beginTransaction(options);
} else {
txn = datastore.beginTransaction();
}
try {
// Check if test already exists in the database
if (!entity.getKind().equalsIgnoreCase(NULL_ENTITY_KIND)) {
try {
if (entity.getKind().equalsIgnoreCase("Test")) {
Entity datastoreEntity = datastore.get(entity.getKey());
TestEntity datastoreTestEntity = TestEntity.fromEntity(datastoreEntity);
if (datastoreTestEntity == null
|| !datastoreTestEntity.equals(entity)) {
entityList.add(entity);
}
} else if (entity.getKind().equalsIgnoreCase("TestPlan")) {
datastore.get(entity.getKey());
} else {
datastore.get(entity.getKey());
}
} catch (EntityNotFoundException e) {
entityList.add(entity);
}
}
datastore.put(txn, entityList);
txn.commit();
break;
} catch (ConcurrentModificationException
| DatastoreFailureException
| DatastoreTimeoutException e) {
entityList.remove(entity);
logger.log(
Level.WARNING,
"Retrying insert kind: " + entity.getKind() + " key: " + entity.getKey());
if (retries++ >= MAX_WRITE_RETRIES) {
logger.log(
Level.SEVERE,
"Exceeded maximum retries kind: "
+ entity.getKind()
+ " key: "
+ entity.getKey());
return false;
}
} finally {
if (txn.isActive()) {
logger.log(
Level.WARNING, "Transaction rollback forced for : " + entity.getKind());
txn.rollback();
}
}
}
return true;
}
}
|
