Periodic sync google spreadsheet data with datastore for excluded API

Test: go/vts-web-staging
Bug: 117810778
Change-Id: Ia08688edbe5ee97c3b57976d7cb6d4a100528755

3 files changed, 425 insertions, 91 deletions

diff --git a/src/main/java/com/android/vts/config/ObjectifyListener.java b/src/main/java/com/android/vts/config/ObjectifyListener.java
index 66f1480..8beeda2 100644
--- a/src/main/java/com/android/vts/config/ObjectifyListener.java
+++ b/src/main/java/com/android/vts/config/ObjectifyListener.java
@@ -19,6 +19,7 @@ package com.android.vts.config;
 import com.android.vts.entity.ApiCoverageEntity;
 import com.android.vts.entity.BranchEntity;
 import com.android.vts.entity.BuildTargetEntity;
+import com.android.vts.entity.ApiCoverageExcludedEntity;
 import com.android.vts.entity.CodeCoverageEntity;
 import com.android.vts.entity.CoverageEntity;
 import com.android.vts.entity.DeviceInfoEntity;
@@ -53,10 +54,7 @@ import java.util.Properties;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 
-
-/**
- * The @WebListener annotation for registering a class as a listener of a web application.
- */
+/** The @WebListener annotation for registering a class as a listener of a web application. */
 @WebListener
 /**
  * Initializing Objectify Service at the container start up before any web components like servlet
@@ -64,97 +62,100 @@ import java.util.logging.Logger;
  */
 public class ObjectifyListener implements ServletContextListener {
 
-  private static final Logger logger = Logger.getLogger(ObjectifyListener.class.getName());
+    private static final Logger logger = Logger.getLogger(ObjectifyListener.class.getName());
 
-  /**
-   * Receives notification that the web application initialization process is starting. This
-   * function will register Entity classes for objectify.
-   */
-  @Override
-  public void contextInitialized(ServletContextEvent servletContextEvent) {
-    ObjectifyService.init();
+    /**
+     * Receives notification that the web application initialization process is starting. This
+     * function will register Entity classes for objectify.
+     */
+    @Override
+    public void contextInitialized(ServletContextEvent servletContextEvent) {
+        ObjectifyService.init();
         ObjectifyService.register(BranchEntity.class);
         ObjectifyService.register(BuildTargetEntity.class);
 
-    ObjectifyService.register(ApiCoverageEntity.class);
-    ObjectifyService.register(CodeCoverageEntity.class);
-    ObjectifyService.register(CoverageEntity.class);
-    ObjectifyService.register(DeviceInfoEntity.class);
-    ObjectifyService.register(TestCoverageStatusEntity.class);
-
-    ObjectifyService.register(ProfilingPointEntity.class);
-    ObjectifyService.register(ProfilingPointRunEntity.class);
-    ObjectifyService.register(ProfilingPointSummaryEntity.class);
-
-    ObjectifyService.register(TestEntity.class);
-    ObjectifyService.register(TestPlanEntity.class);
-    ObjectifyService.register(TestPlanRunEntity.class);
-    ObjectifyService.register(TestRunEntity.class);
-    ObjectifyService.register(TestCaseRunEntity.class);
-    ObjectifyService.register(TestStatusEntity.class);
-    ObjectifyService.register(TestSuiteFileEntity.class);
-    ObjectifyService.register(TestSuiteResultEntity.class);
-    ObjectifyService.register(RoleEntity.class);
-    ObjectifyService.register(UserEntity.class);
-    ObjectifyService.begin();
-    logger.log(Level.INFO, "Value Initialized from context.");
-
-    Properties systemConfigProp = new Properties();
-
-    try {
-      InputStream defaultInputStream =
-          ObjectifyListener.class
-              .getClassLoader()
-              .getResourceAsStream("config.properties");
-
-      systemConfigProp.load(defaultInputStream);
-
-      String roleList = systemConfigProp.getProperty("user.roleList");
-      Supplier<Stream<String>> streamSupplier = () -> Arrays.stream(roleList.split(","));
-      this.createRoles(streamSupplier.get());
-
-      String adminEmail = systemConfigProp.getProperty("user.adminEmail");
-      if (adminEmail.isEmpty()) {
-        logger.log(Level.WARNING, "Admin email is not properly set. Check config file");
-      } else {
-        String adminName = systemConfigProp.getProperty("user.adminName");
-        String adminCompany = systemConfigProp.getProperty("user.adminCompany");
-        Optional<String> roleName = streamSupplier.get().filter(r -> r.equals("admin")).findFirst();
-        this.createAdminUser(adminEmail, adminName, adminCompany, roleName.orElse("admin"));
-      }
-    } catch (FileNotFoundException e) {
-      e.printStackTrace();
-    } catch (IOException e) {
-      e.printStackTrace();
+        ObjectifyService.register(ApiCoverageEntity.class);
+        ObjectifyService.register(ApiCoverageExcludedEntity.class);
+        ObjectifyService.register(CodeCoverageEntity.class);
+        ObjectifyService.register(CoverageEntity.class);
+        ObjectifyService.register(DeviceInfoEntity.class);
+        ObjectifyService.register(TestCoverageStatusEntity.class);
+
+        ObjectifyService.register(ProfilingPointEntity.class);
+        ObjectifyService.register(ProfilingPointRunEntity.class);
+        ObjectifyService.register(ProfilingPointSummaryEntity.class);
+
+        ObjectifyService.register(TestEntity.class);
+        ObjectifyService.register(TestPlanEntity.class);
+        ObjectifyService.register(TestPlanRunEntity.class);
+        ObjectifyService.register(TestRunEntity.class);
+        ObjectifyService.register(TestCaseRunEntity.class);
+        ObjectifyService.register(TestStatusEntity.class);
+        ObjectifyService.register(TestSuiteFileEntity.class);
+        ObjectifyService.register(TestSuiteResultEntity.class);
+        ObjectifyService.register(RoleEntity.class);
+        ObjectifyService.register(UserEntity.class);
+        ObjectifyService.begin();
+        logger.log(Level.INFO, "Value Initialized from context.");
+
+        Properties systemConfigProp = new Properties();
+
+        try {
+            InputStream defaultInputStream =
+                    ObjectifyListener.class
+                            .getClassLoader()
+                            .getResourceAsStream("config.properties");
+
+            systemConfigProp.load(defaultInputStream);
+
+            String roleList = systemConfigProp.getProperty("user.roleList");
+            Supplier<Stream<String>> streamSupplier = () -> Arrays.stream(roleList.split(","));
+            this.createRoles(streamSupplier.get());
+
+            String adminEmail = systemConfigProp.getProperty("user.adminEmail");
+            if (adminEmail.isEmpty()) {
+                logger.log(Level.WARNING, "Admin email is not properly set. Check config file");
+            } else {
+                String adminName = systemConfigProp.getProperty("user.adminName");
+                String adminCompany = systemConfigProp.getProperty("user.adminCompany");
+                Optional<String> roleName =
+                        streamSupplier.get().filter(r -> r.equals("admin")).findFirst();
+                this.createAdminUser(adminEmail, adminName, adminCompany, roleName.orElse("admin"));
+            }
+        } catch (FileNotFoundException e) {
+            e.printStackTrace();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+    }
+
+    /** Receives notification that the ServletContext is about to be shut down. */
+    @Override
+    public void contextDestroyed(ServletContextEvent servletContextEvent) {
+        ServletContext servletContext = servletContextEvent.getServletContext();
+        logger.log(Level.INFO, "Value deleted from context.");
+    }
+
+    private void createRoles(Stream<String> roleStream) {
+        roleStream
+                .map(role -> role.trim())
+                .forEach(
+                        roleName -> {
+                            RoleEntity roleEntity = new RoleEntity(roleName);
+                            roleEntity.save();
+                        });
     }
-  }
-
-  /**
-   * Receives notification that the ServletContext is about to be shut down.
-   */
-  @Override
-  public void contextDestroyed(ServletContextEvent servletContextEvent) {
-    ServletContext servletContext = servletContextEvent.getServletContext();
-    logger.log(Level.INFO, "Value deleted from context.");
-  }
-
-  private void createRoles(Stream<String> roleStream) {
-    roleStream.map(role -> role.trim()).forEach(roleName -> {
-      RoleEntity roleEntity = new RoleEntity(roleName);
-      roleEntity.save();
-    });
-  }
-
-  private void createAdminUser(String email, String name, String company, String role) {
-    Optional<UserEntity> adminUserEntityOptional = Optional
-        .ofNullable(UserEntity.getAdminUser(email));
-    if (adminUserEntityOptional.isPresent()) {
-      logger.log(Level.INFO, "The user is already registered.");
-    } else {
-      UserEntity userEntity = new UserEntity(email, name, company, role);
-      userEntity.setIsAdmin(true);
-      userEntity.save();
-      logger.log(Level.INFO, "The user is saved successfully.");
+
+    private void createAdminUser(String email, String name, String company, String role) {
+        Optional<UserEntity> adminUserEntityOptional =
+                Optional.ofNullable(UserEntity.getAdminUser(email));
+        if (adminUserEntityOptional.isPresent()) {
+            logger.log(Level.INFO, "The user is already registered.");
+        } else {
+            UserEntity userEntity = new UserEntity(email, name, company, role);
+            userEntity.setIsAdmin(true);
+            userEntity.save();
+            logger.log(Level.INFO, "The user is saved successfully.");
+        }
     }
-  }
 }
diff --git a/src/main/java/com/android/vts/entity/ApiCoverageExcludedEntity.java b/src/main/java/com/android/vts/entity/ApiCoverageExcludedEntity.java
new file mode 100644
index 0000000..834f8cc
--- /dev/null
+++ b/src/main/java/com/android/vts/entity/ApiCoverageExcludedEntity.java
@@ -0,0 +1,168 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.android.vts.entity;
+
+import com.fasterxml.jackson.annotation.JsonAutoDetect;
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.googlecode.objectify.Key;
+import com.googlecode.objectify.annotation.Cache;
+import com.googlecode.objectify.annotation.Entity;
+import com.googlecode.objectify.annotation.Id;
+import com.googlecode.objectify.annotation.Index;
+import lombok.EqualsAndHashCode;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+import java.util.Collection;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
+
+import static com.googlecode.objectify.ObjectifyService.ofy;
+
+/**
+ * This entity class contain the excluded API information. And this information will be used to
+ * calculate more precise the ratio of API coverage.
+ */
+@Cache
+@Entity(name = "ApiCoverageExcluded")
+@EqualsAndHashCode(of = "id")
+@NoArgsConstructor
+@JsonAutoDetect(fieldVisibility = JsonAutoDetect.Visibility.ANY)
+@JsonIgnoreProperties({"id", "parent"})
+public class ApiCoverageExcludedEntity {
+
+    // The maximum number of entity list size to insert datastore
+    private static final int maxNumEntitySize = 500;
+
+    /** ApiCoverageEntity id field */
+    @Id @Getter @Setter private String id;
+
+    /** Package name. e.g. android.hardware.foo. */
+    @Index @Getter @Setter private String packageName;
+
+    /** Major Version. e.g. 1, 2. */
+    @Getter @Setter private int majorVersion;
+
+    /** Minor Version. e.g. 0. */
+    @Getter @Setter private int minorVersion;
+
+    /** Interface name. e.g. IFoo. */
+    @Index @Getter @Setter private String interfaceName;
+
+    /** API Name */
+    @Index @Getter @Setter private String apiName;
+
+    /** The reason comment for the excluded API */
+    @Getter @Setter private String comment;
+
+    /** When this record was created or updated */
+    @Index @Getter @Setter private Date updated;
+
+    /** Constructor function for ApiCoverageExcludedEntity Class */
+    public ApiCoverageExcludedEntity(
+            String packageName,
+            String version,
+            String interfaceName,
+            String apiName,
+            String comment) {
+
+        this.packageName = packageName;
+        this.interfaceName = interfaceName;
+        this.apiName = apiName;
+        this.comment = comment;
+
+        this.setVersions(version);
+    }
+
+    /** Setting major and minor version from version string */
+    private void setVersions(String version) {
+        String[] versionArray = version.split("[.]");
+        if (versionArray.length == 0) {
+            this.majorVersion = 0;
+            this.minorVersion = 0;
+        } else if (versionArray.length == 1) {
+            this.majorVersion = Integer.parseInt(versionArray[0]);
+            this.minorVersion = 0;
+        } else {
+            this.majorVersion = Integer.parseInt(versionArray[0]);
+            this.minorVersion = Integer.parseInt(versionArray[1]);
+        }
+    }
+
+    /** Getting objectify ID from the entity information */
+    private String getObjectifyId() {
+        return this.packageName
+                + "."
+                + this.majorVersion
+                + "."
+                + this.minorVersion
+                + "."
+                + this.interfaceName
+                + "."
+                + this.apiName;
+    }
+
+    /** Getting key from the entity */
+    public Key<ApiCoverageExcludedEntity> getKey() {
+        return Key.create(ApiCoverageExcludedEntity.class, this.getObjectifyId());
+    }
+
+    /** Saving function for the instance of this class */
+    public Key<ApiCoverageExcludedEntity> save() {
+        this.id = this.getObjectifyId();
+        this.updated = new Date();
+        return ofy().save().entity(this).now();
+    }
+
+    /** Spliting a list based on a given size */
+    public static <T> Collection<List<T>> partitionBasedOnSize(List<T> inputList, int size) {
+        final AtomicInteger counter = new AtomicInteger(0);
+        return inputList
+                .stream()
+                .collect(Collectors.groupingBy(s -> counter.getAndIncrement() / size))
+                .values();
+    }
+
+    /** Saving function with parameter of this entity List */
+    public static void saveAll(List<ApiCoverageExcludedEntity> apiCoverageExcludedEntityList) {
+        List<ApiCoverageExcludedEntity> entityWithIdList =
+                apiCoverageExcludedEntityList
+                        .stream()
+                        .map(
+                                entity -> {
+                                    entity.setId(entity.getObjectifyId());
+                                    entity.setUpdated(new Date());
+                                    return entity;
+                                })
+                        .collect(Collectors.toList());
+
+        partitionBasedOnSize(entityWithIdList, maxNumEntitySize)
+                .stream()
+                .forEach(
+                        entityList -> {
+                            ofy().save().entities(entityList).now();
+                        });
+    }
+
+    /** Get All Key List of ApiCoverageExcludedEntity */
+    public static List<Key<ApiCoverageExcludedEntity>> getAllKeyList() {
+        return ofy().load().type(ApiCoverageExcludedEntity.class).keys().list();
+    }
+}
diff --git a/src/main/java/com/android/vts/job/VtsSpreadSheetSyncServlet.java b/src/main/java/com/android/vts/job/VtsSpreadSheetSyncServlet.java
new file mode 100644
index 0000000..079aa2d
--- /dev/null
+++ b/src/main/java/com/android/vts/job/VtsSpreadSheetSyncServlet.java
@@ -0,0 +1,165 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you
+ * may not use this file except in compliance with the License. You may
+ * obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ * implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+package com.android.vts.job;
+
+import com.android.vts.entity.ApiCoverageExcludedEntity;
+import com.google.api.client.auth.oauth2.Credential;
+import com.google.api.client.extensions.java6.auth.oauth2.AuthorizationCodeInstalledApp;
+import com.google.api.client.extensions.jetty.auth.oauth2.LocalServerReceiver;
+import com.google.api.client.googleapis.auth.oauth2.GoogleAuthorizationCodeFlow;
+import com.google.api.client.googleapis.auth.oauth2.GoogleClientSecrets;
+import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport;
+import com.google.api.client.http.javanet.NetHttpTransport;
+import com.google.api.client.json.JsonFactory;
+import com.google.api.client.json.jackson2.JacksonFactory;
+import com.google.api.client.util.store.FileDataStoreFactory;
+import com.google.api.services.sheets.v4.Sheets;
+import com.google.api.services.sheets.v4.SheetsScopes;
+import com.google.api.services.sheets.v4.model.ValueRange;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import javax.servlet.ServletConfig;
+import javax.servlet.ServletException;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/** Job to sync excluded API data in google spreadsheet with datastore's entity. */
+public class VtsSpreadSheetSyncServlet extends BaseJobServlet {
+
+    protected static final Logger logger =
+            Logger.getLogger(VtsSpreadSheetSyncServlet.class.getName());
+
+    private static final String APPLICATION_NAME = "VTS Dashboard";
+    private static final JsonFactory JSON_FACTORY = JacksonFactory.getDefaultInstance();
+    private static final String TOKENS_DIRECTORY_PATH = "tokens";
+
+    /**
+     * Global instance of the scopes. If modifying these scopes, delete your previously saved
+     * tokens/ folder.
+     */
+    private static final List<String> SCOPES =
+            Collections.singletonList(SheetsScopes.SPREADSHEETS_READONLY);
+
+    private String CREDENTIALS_KEY_FILE = "";
+
+    /** GoogleClientSecrets for GoogleAuthorizationCodeFlow Builder */
+    private GoogleClientSecrets clientSecrets;
+
+    /** This is the ID of google spreadsheet. */
+    private String SPREAD_SHEET_ID = "";
+
+    /** This is the range to read of google spreadsheet. */
+    private String SPREAD_SHEET_RANGE = "";
+
+    @Override
+    public void init(ServletConfig servletConfig) throws ServletException {
+        super.init(servletConfig);
+
+        try {
+            CREDENTIALS_KEY_FILE = systemConfigProp.getProperty("api.coverage.keyFile");
+            SPREAD_SHEET_ID = systemConfigProp.getProperty("api.coverage.spreadSheetId");
+            SPREAD_SHEET_RANGE = systemConfigProp.getProperty("api.coverage.spreadSheetRange");
+
+            InputStream keyFileInputStream =
+                    this.getClass()
+                            .getClassLoader()
+                            .getResourceAsStream("keys/" + CREDENTIALS_KEY_FILE);
+            InputStreamReader keyFileStreamReader = new InputStreamReader(keyFileInputStream);
+
+            this.clientSecrets = GoogleClientSecrets.load(JSON_FACTORY, keyFileStreamReader);
+        } catch (IOException ioe) {
+            logger.log(Level.SEVERE, ioe.getMessage());
+        } catch (Exception exception) {
+            logger.log(Level.SEVERE, exception.getMessage());
+        }
+    }
+
+    /**
+     * Creates an authorized Credential object.
+     *
+     * @param HTTP_TRANSPORT The network HTTP Transport.
+     * @return An authorized Credential object.
+     * @throws IOException If the credentials.json file cannot be found.
+     */
+    private Credential getCredentials(final NetHttpTransport HTTP_TRANSPORT) throws IOException {
+
+        // Build flow and trigger user authorization request.
+        File fileTokenDirPath = new File(TOKENS_DIRECTORY_PATH);
+        FileDataStoreFactory fileDataStoreFactory = new FileDataStoreFactory(fileTokenDirPath);
+
+        GoogleAuthorizationCodeFlow flow =
+                new GoogleAuthorizationCodeFlow.Builder(
+                                HTTP_TRANSPORT, JSON_FACTORY, this.clientSecrets, SCOPES)
+                        .setDataStoreFactory(fileDataStoreFactory)
+                        .setAccessType("offline")
+                        .build();
+        LocalServerReceiver localServerReceiver = new LocalServerReceiver();
+        return new AuthorizationCodeInstalledApp(flow, localServerReceiver).authorize("user");
+    }
+
+    @Override
+    public void doGet(HttpServletRequest request, HttpServletResponse response) throws IOException {
+
+        try {
+            // Build a new authorized API client service.
+            final NetHttpTransport HTTP_TRANSPORT = GoogleNetHttpTransport.newTrustedTransport();
+            Sheets service =
+                    new Sheets.Builder(HTTP_TRANSPORT, JSON_FACTORY, getCredentials(HTTP_TRANSPORT))
+                            .setApplicationName(APPLICATION_NAME)
+                            .build();
+
+            ValueRange valueRange =
+                    service.spreadsheets()
+                            .values()
+                            .get(SPREAD_SHEET_ID, SPREAD_SHEET_RANGE)
+                            .execute();
+
+            List<ApiCoverageExcludedEntity> apiCoverageExcludedEntities = new ArrayList<>();
+            List<List<Object>> values = valueRange.getValues();
+            if (values == null || values.isEmpty()) {
+                logger.log(Level.WARNING, "No data found in google spreadsheet.");
+            } else {
+                for (List row : values) {
+                    ApiCoverageExcludedEntity apiCoverageExcludedEntity =
+                            new ApiCoverageExcludedEntity(
+                                    row.get(0).toString(),
+                                    row.get(1).toString(),
+                                    row.get(2).toString(),
+                                    row.get(3).toString(),
+                                    row.get(4).toString());
+                    apiCoverageExcludedEntities.add(apiCoverageExcludedEntity);
+                }
+            }
+
+            ApiCoverageExcludedEntity.saveAll(apiCoverageExcludedEntities);
+
+        } catch (GeneralSecurityException gse) {
+            logger.log(Level.SEVERE, gse.getMessage());
+        }
+    }
+}