Prevent indexing of all files in project

before sync completes.

When the user opens a large project that was previously built with
Gradle it usually contains multiple `build` directories with many files.

These files in `build` directories get automatically added to the VFS
while the project is being scanned before indexing and, moreover, these
files get indexed.

Many files in `build` directories change at each build.

Indexing is a one time operation and once sync completes and `build`
directories become excluded from the project they are not indexed
anymore. However, the records of these files in the VFS need to be
update each time files change. It causes significant slowness and
freezes when building large projects.

Prevent files being indexed and added to the VFS before Gradle sync
complete by temporary excluding all currently existing subdirectories
and some well known directories that can appear while syncing/building
from the project indexing scope.

Bug: 200820556
Test: PlatformIntegrationTest
Change-Id: I8aebb552887bd80ccbd7c1548e3f8aeaa5b49c2b

4 files changed, 103 insertions, 7 deletions

diff --git a/android/src/com/android/tools/idea/gradle/project/importing/TopLevelModuleFactory.kt b/android/src/com/android/tools/idea/gradle/project/importing/TopLevelModuleFactory.kt
index 909faed6312..0e8208da518 100644
--- a/android/src/com/android/tools/idea/gradle/project/importing/TopLevelModuleFactory.kt
+++ b/android/src/com/android/tools/idea/gradle/project/importing/TopLevelModuleFactory.kt
@@ -18,6 +18,7 @@ package com.android.tools.idea.gradle.project.importing
 import com.android.SdkConstants
 import com.android.tools.idea.IdeInfo
 import com.android.tools.idea.Projects
+import com.android.tools.idea.gradle.project.GradleProjectInfo
 import com.android.tools.idea.gradle.project.facet.gradle.GradleFacet
 import com.android.tools.idea.gradle.util.GradleUtil
 import com.intellij.facet.FacetManager
@@ -25,15 +26,19 @@ import com.intellij.openapi.diagnostic.Logger
 import com.intellij.openapi.externalSystem.ExternalSystemModulePropertyManager.Companion.getInstance
 import com.intellij.openapi.externalSystem.model.project.ModuleData
 import com.intellij.openapi.externalSystem.model.project.ProjectData
+import com.intellij.openapi.externalSystem.service.project.manage.ProjectDataImportListener
 import com.intellij.openapi.externalSystem.util.ExternalSystemApiUtil
 import com.intellij.openapi.externalSystem.util.ExternalSystemApiUtil.toCanonicalPath
-import com.intellij.openapi.module.Module
 import com.intellij.openapi.module.ModuleManager
 import com.intellij.openapi.module.ModuleWithNameAlreadyExists
 import com.intellij.openapi.module.StdModuleTypes
 import com.intellij.openapi.project.Project
 import com.intellij.openapi.roots.ModuleRootManager
+import com.intellij.openapi.roots.ProjectRootManager
+import com.intellij.openapi.roots.impl.DirectoryIndexExcludePolicy
+import com.intellij.openapi.util.Key
 import com.intellij.openapi.vfs.VfsUtil
+import com.intellij.openapi.vfs.VfsUtilCore
 import com.intellij.util.PathUtil
 import org.jetbrains.plugins.gradle.settings.GradleSettings
 import org.jetbrains.plugins.gradle.util.GradleConstants
@@ -121,7 +126,9 @@ class TopLevelModuleFactory() {
       )
     val model = ModuleRootManager.getInstance(module).modifiableModel
 
-    (model.contentEntries.singleOrNull() ?: model.addContentEntry(gradleRootVirtualFile))
+    if (model.contentEntries.singleOrNull() == null) {
+      model.addContentEntry(gradleRootVirtualFile)
+    }
     if (IdeInfo.getInstance().isAndroidStudio) {
       // If sync fails, make sure that the project has a JDK, otherwise Groovy indices won't work (a common scenario where
       // users will update build.gradle files to fix Gradle sync.)
@@ -146,4 +153,4 @@ class TopLevelModuleFactory() {
   }
 }
 
-private val LOG = Logger.getInstance(TopLevelModuleFactory::class.java)
+private val LOG = Logger.getInstance(TopLevelModuleFactory::class.java)
\ No newline at end of file
diff --git a/project-system-gradle/src/META-INF/project-system-gradle-plugin-androidstudio.xml b/project-system-gradle/src/META-INF/project-system-gradle-plugin-androidstudio.xml
index 3b073a2c936..b7f7abcbe5c 100644
--- a/project-system-gradle/src/META-INF/project-system-gradle-plugin-androidstudio.xml
+++ b/project-system-gradle/src/META-INF/project-system-gradle-plugin-androidstudio.xml
@@ -26,6 +26,7 @@
       implementation="com.android.tools.idea.gradle.project.importing.AndroidGradleProjectConfigurator"
       order="after PlatformProjectConfigurator"
     />
+    <directoryIndexExcludePolicy implementation="com.android.tools.idea.gradle.project.importing.InitialImportExcludeDirectoryPolicy" />
   </extensions>
   <actions>
     <action id="WelcomeScreen.Configure.ProjectStructure" class="com.android.tools.idea.gradle.actions.AndroidTemplateProjectStructureAction" overrides="true" />
diff --git a/project-system-gradle/src/com/android/tools/idea/gradle/project/importing/InitialImportExcludeDirectoryPolicy.kt b/project-system-gradle/src/com/android/tools/idea/gradle/project/importing/InitialImportExcludeDirectoryPolicy.kt
new file mode 100644
index 00000000000..fb6435ef538
--- /dev/null
+++ b/project-system-gradle/src/com/android/tools/idea/gradle/project/importing/InitialImportExcludeDirectoryPolicy.kt
@@ -0,0 +1,88 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.android.tools.idea.gradle.project.importing
+
+import com.android.tools.idea.gradle.project.GradleProjectInfo
+import com.intellij.openapi.project.Project
+import com.intellij.openapi.roots.ProjectRootManager
+import com.intellij.openapi.roots.impl.DirectoryIndexExcludePolicy
+import com.intellij.openapi.util.Key
+import com.intellij.openapi.vfs.VfsUtilCore
+import org.jetbrains.plugins.gradle.settings.GradleSettings
+import java.io.File
+
+/**
+ * A [DirectoryIndexExcludePolicy] that temporarily excludes all directories under a Gradle root from the project while the initial sync
+ * runs.
+ *
+ * This is needed to prevent directories that are normally excluded from the project after sync from being indexed or included in the VFS
+ * before the initial sync completes. Otherwise, indexing itself may take much time and any file included in the VFS stays there and is
+ * updated when the VFS area containing this file is refreshed. This in turn may result it massive VFS updates after building a project if
+ * its build directories end up in the VFS.
+ *
+ * [InitialImportExcludeDirectoryPolicy] reports any directories directly under the project's Gradle roots as excluded while there is no
+ * source roots yet in the project (i.e. sync has not yet succeeded).
+ */
+class InitialImportExcludeDirectoryPolicy(private val project: Project) : DirectoryIndexExcludePolicy {
+  override fun getExcludeUrlsForProject(): Array<out String> {
+    if (project.getUserData(EXCLUDE_DIRS_KEY) == false) return emptyArray()
+    // Stop returning any exclude directories when sync succeeds. A successful sync sets up some source roots unless all modules are empty.
+    // Note: We cannot rely on listeners here to clear the flag as roots are enumerated just after committing project model changes and
+    //       listeners run too late.
+    if (!project.isGradleProject() || project.projectHasSourceRoots()) {
+      project.putUserData(EXCLUDE_DIRS_KEY, false)
+      return emptyArray()
+    }
+
+    return project
+      .getGradleRoots()
+      // TODO(b/242440055): Multiple Gradle roots are not yet supported here.
+      .flatMap { getDirectoriesToExcludeUnderGradleRoot(it) }
+      .map { VfsUtilCore.pathToUrl(it.path) }
+      .toTypedArray()
+  }
+}
+
+private fun Project.isGradleProject(): Boolean {
+  return GradleProjectInfo.getInstance(this).isBuildWithGradle
+}
+
+private fun Project.projectHasSourceRoots(): Boolean {
+  return ProjectRootManager.getInstance(this).orderEntries().withoutLibraries().withoutSdk().sources().urls.isNotEmpty()
+}
+
+private fun Project.getGradleRoots(): List<File> {
+  return GradleSettings.getInstance(this)
+    .linkedProjectsSettings
+    .mapNotNull { File(it.externalProjectPath) }
+}
+
+private fun getDirectoriesToExcludeUnderGradleRoot(gradleRoot: File): List<File> {
+  val existingDirectoryNames =
+    gradleRoot
+      .listFiles()
+      .orEmpty()
+      .filter { it.isDirectory }
+      .map { it.name }
+      .toSet()
+
+  val wellKnownDirectoryNames = setOf("build", ".gradle")
+
+  return (existingDirectoryNames + wellKnownDirectoryNames)
+    .map { gradleRoot.resolve(it) }
+}
+
+private val EXCLUDE_DIRS_KEY: Key<Boolean> = Key.create("temporary_excluded_dirs")
\ No newline at end of file
diff --git a/project-system-gradle/testSrc/com/android/tools/idea/gradle/project/sync/PlatformIntegrationTest.kt b/project-system-gradle/testSrc/com/android/tools/idea/gradle/project/sync/PlatformIntegrationTest.kt
index 6553724f082..55f21dd5872 100644
--- a/project-system-gradle/testSrc/com/android/tools/idea/gradle/project/sync/PlatformIntegrationTest.kt
+++ b/project-system-gradle/testSrc/com/android/tools/idea/gradle/project/sync/PlatformIntegrationTest.kt
@@ -56,7 +56,9 @@ import com.intellij.openapi.util.io.FileUtil.toSystemIndependentName
 import com.intellij.openapi.vfs.VfsUtil
 import com.intellij.openapi.vfs.VirtualFile
 import com.intellij.openapi.vfs.newvfs.NewVirtualFile
+import com.intellij.platform.DirectoryProjectConfigurator
 import com.intellij.task.ProjectTaskManager
+import com.intellij.util.application
 import org.jetbrains.annotations.SystemIndependent
 import org.junit.Rule
 import org.junit.Test
@@ -123,8 +125,7 @@ class PlatformIntegrationTest : GradleIntegrationTest {
     openPreparedProject("copy") { project ->
       expect.that(copy.resolve("app/build/intermediates/dex/debug").exists())
       expect.that(copy.resolveVirtualIfCached("app/build")).isNotNull()
-      // TODO(b/200820556): Most files in `build` dir should not be added to the VFS.
-      // TODO(b/200820556): expect.that(copy.resolveVirtualIfCached("app/build/intermediates/dex/debug")).isNull()
+      expect.that(copy.resolveVirtualIfCached("app/build/intermediates/dex/debug")).isNull()
     }
   }
 
@@ -147,8 +148,7 @@ class PlatformIntegrationTest : GradleIntegrationTest {
     openPreparedProject("copy") { project ->
       expect.that(copy.resolve("app/build/intermediates/dex/debug").exists())
       expect.that(copy.resolveVirtualIfCached("app/build")).isNotNull()
-      // TODO(b/200820556): Most files in `build` dir should not be added to the VFS.
-      // TODO(b/200820556): expect.that(copy.resolveVirtualIfCached("app/build/intermediates/dex/debug")).isNull()
+      expect.that(copy.resolveVirtualIfCached("app/build/intermediates/dex/debug")).isNull()
     }
   }