Fix conflicts of dokka core dependency between agp and other plugins

With this change, we use dokka-core as compileOnly dependency of AGP and
remove it from the buildScript runtime classpath. At the same time, we
provide dokka-core dependency via worker classloader. Therefore, the
dokka-core that we use wouldn't override other dokka-core version used
by other plugins(e.g. dokka gradle plugin).

Due to the thread safety issue b/211725171, we limit the worker number
to one for JavaDocGenerationTask. We could revist this thread safety
issue later and remove the limit if our corresponding integration
tests don't have flakiness.

This CL doesn' bump up the dokka-core version to limit the scope of this
change.

Bug: 229979216
Test: GmavenZipTest
Change-Id: Idb53c76ff0be57641cb194a01b010f57baee4bb9
(cherry picked from commit bd824c89229deaa03101064b765da1d2e94ee414)

5 files changed, 168 insertions, 83 deletions

diff --git a/build-system/gradle-core/build.gradle b/build-system/gradle-core/build.gradle
index a736111051..b581c7a50b 100644
--- a/build-system/gradle-core/build.gradle
+++ b/build-system/gradle-core/build.gradle
@@ -169,11 +169,11 @@ dependencies {
     implementation libs.grpc_stub
     implementation libs.tink
     implementation libs.unified_test_platform
-    implementation libs.dokka_core
 
     compileOnly libs.kotlin_gradle_plugin
     compileOnly libs.jacoco_core
     compileOnly libs.jacoco_report
+    compileOnly libs.dokka_core
     // already packaged in (':base:builder')
     compileOnly project(':base:profile')
 
diff --git a/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/LibraryTaskManager.java b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/LibraryTaskManager.java
index 9caa847ae1..b87df0aca8 100644
--- a/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/LibraryTaskManager.java
+++ b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/LibraryTaskManager.java
@@ -49,6 +49,7 @@ import com.android.build.gradle.internal.res.GenerateApiPublicTxtTask;
 import com.android.build.gradle.internal.res.GenerateEmptyResourceFilesTask;
 import com.android.build.gradle.internal.scope.BuildFeatureValues;
 import com.android.build.gradle.internal.scope.InternalArtifactType;
+import com.android.build.gradle.internal.services.DokkaParallelBuildService;
 import com.android.build.gradle.internal.tasks.AarMetadataTask;
 import com.android.build.gradle.internal.tasks.BundleLibraryClassesDir;
 import com.android.build.gradle.internal.tasks.BundleLibraryClassesJar;
@@ -369,6 +370,7 @@ public class LibraryTaskManager extends TaskManager<LibraryVariantBuilderImpl, L
                 taskFactory.register(new SourceJarTask.CreationAction(variant));
             }
             if (components.stream().anyMatch(ComponentPublishingInfo::getWithJavadocJar)) {
+                new DokkaParallelBuildService.RegistrationAction(project).execute();
                 taskFactory.register(new JavaDocGenerationTask.CreationAction(variant));
                 taskFactory.register(new JavaDocJarTask.CreationAction(variant));
             }
diff --git a/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/services/DokkaParallelBuildService.kt b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/services/DokkaParallelBuildService.kt
new file mode 100644
index 0000000000..7bd35a6bbf
--- /dev/null
+++ b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/services/DokkaParallelBuildService.kt
@@ -0,0 +1,38 @@
+/*
+ * Copyright (C) 2022 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.android.build.gradle.internal.services
+
+import org.gradle.api.Project
+import org.gradle.api.services.BuildService
+import org.gradle.api.services.BuildServiceParameters
+
+/**
+ * [DokkaParallelBuildService] is to limit the number of workers in Javadoc generation task to one
+ * in order to avoid thread safety issue in dokka-core. See https://github.com/Kotlin/dokka/issues/2308
+ */
+abstract class DokkaParallelBuildService : BuildService<BuildServiceParameters.None> {
+    class RegistrationAction(project: Project) :
+        ServiceRegistrationAction<DokkaParallelBuildService, BuildServiceParameters.None>(
+            project,
+            DokkaParallelBuildService::class.java,
+            MAX_WORKER_NUMBER
+        ) {
+        override fun configure(parameters: BuildServiceParameters.None) {}
+    }
+}
+
+private const val MAX_WORKER_NUMBER = 1
diff --git a/build-system/gradle-core/src/main/java/com/android/build/gradle/tasks/JavaDocGenerationTask.kt b/build-system/gradle-core/src/main/java/com/android/build/gradle/tasks/JavaDocGenerationTask.kt
index 8cde38dad0..b258270099 100644
--- a/build-system/gradle-core/src/main/java/com/android/build/gradle/tasks/JavaDocGenerationTask.kt
+++ b/build-system/gradle-core/src/main/java/com/android/build/gradle/tasks/JavaDocGenerationTask.kt
@@ -20,12 +20,15 @@ import com.android.build.gradle.internal.component.ComponentCreationConfig
 import com.android.build.gradle.internal.publishing.AndroidArtifacts.ArtifactType.CLASSES_JAR
 import com.android.build.gradle.internal.publishing.AndroidArtifacts.ConsumedConfigType.COMPILE_CLASSPATH
 import com.android.build.gradle.internal.scope.InternalArtifactType
+import com.android.build.gradle.internal.services.DokkaParallelBuildService
+import com.android.build.gradle.internal.services.getBuildService
 import com.android.build.gradle.internal.tasks.NonIncrementalTask
 import com.android.build.gradle.internal.tasks.factory.VariantTaskCreationAction
 import com.android.build.gradle.internal.utils.fromDisallowChanges
 import com.android.build.gradle.internal.utils.setDisallowChanges
 import org.gradle.api.file.ConfigurableFileCollection
 import org.gradle.api.file.DirectoryProperty
+import org.gradle.api.logging.Logging
 import org.gradle.api.provider.Property
 import org.gradle.api.tasks.CacheableTask
 import org.gradle.api.tasks.Classpath
@@ -36,6 +39,8 @@ import org.gradle.api.tasks.PathSensitive
 import org.gradle.api.tasks.PathSensitivity
 import org.gradle.api.tasks.TaskAction
 import org.gradle.api.tasks.TaskProvider
+import org.gradle.workers.WorkAction
+import org.gradle.workers.WorkParameters
 import org.jetbrains.dokka.DokkaBootstrapImpl
 import org.jetbrains.dokka.DokkaConfiguration.ExternalDocumentationLink
 import org.jetbrains.dokka.DokkaConfigurationImpl
@@ -79,97 +84,129 @@ abstract class JavaDocGenerationTask : NonIncrementalTask() {
     @get:Classpath
     abstract val dokkaRuntimeClasspath: ConfigurableFileCollection
 
+    @get:Classpath
+    abstract val dokkaCoreClasspath: ConfigurableFileCollection
+
     @TaskAction
     override fun doTaskAction() {
-        val dokkaConfiguration =
-            buildDokkaConfiguration(
-                dokkaPlugins,
-                sources,
-                classpath,
-                outputDirectory,
-                path,
-                projectPath.get(),
-                moduleVersion.get())
-
-        getRuntimeClassLoader(dokkaRuntimeClasspath).use {
-            val bootstrapClass = it.loadClass(DokkaBootstrapImpl::class.qualifiedName)
-            val bootstrapInstance = bootstrapClass.constructors.first().newInstance()
-            val configureMethod =
-                bootstrapClass.getMethod("configure", String::class.java, BiConsumer::class.java)
-            val generateMethod = bootstrapClass.getMethod("generate")
-
-            configureMethod.invoke(
-                bootstrapInstance, dokkaConfiguration.toJsonString(), createProxyLogger())
-            generateMethod.invoke(bootstrapInstance)
+        workerExecutor.classLoaderIsolation{ it.classpath.from(dokkaCoreClasspath) }.submit(
+            DokkaWorkAction::class.java
+        ) {
+            it.dokkaPlugins.from(dokkaPlugins)
+            it.sources.from(sources)
+            it.classpath.from(classpath)
+            it.outputDirectory.set(outputDirectory)
+            it.path.set(path)
+            it.moduleVersion.set(moduleVersion)
+            it.dokkaRuntimeClasspath.from(dokkaRuntimeClasspath)
+            it.projectPath.set(this.projectPath)
         }
     }
 
-    /**
-     * Dokka needs a separate classpath because it not only uses classes bundled from the kotlin
-     * compiler, but also classes from intellij that would heavily interfere with other plugins
-     * when loaded directly.
-     *
-     * Given Gradle leaks kotlin-stdlib to classpath of workers running in classloader isolation
-     * and process isolation mode, we shouldn't launch Dokka via Gradle workers. Instead,
-     * we need to load Dokka runtime dependencies manually and launch it using reflection.
-     */
-    private fun getRuntimeClassLoader(runtime: ConfigurableFileCollection): URLClassLoader {
-        val runtimeJars = runtime.files
-        return URLClassLoader(
-            runtimeJars.map { it.toURI().toURL() }.toTypedArray(),
-            ClassLoader.getSystemClassLoader().parent
-        )
-    }
+    abstract class DokkaWorkAction : WorkAction<DokkaWorkAction.Params> {
+        abstract class Params: WorkParameters {
+            abstract val dokkaPlugins: ConfigurableFileCollection
+            abstract val sources: ConfigurableFileCollection
+            abstract val classpath: ConfigurableFileCollection
+            abstract val outputDirectory: DirectoryProperty
+            abstract val path: Property<String>
+            abstract val moduleVersion: Property<String>
+            abstract val dokkaRuntimeClasspath: ConfigurableFileCollection
+            abstract val projectPath: Property<String>
+        }
 
-    private fun buildDokkaConfiguration(
-        plugins: ConfigurableFileCollection,
-        sources: ConfigurableFileCollection,
-        classpath: ConfigurableFileCollection,
-        outputDirectory: DirectoryProperty,
-        taskPath: String,
-        moduleName: String,
-        moduleVersion: String
-    ): DokkaConfigurationImpl {
-        return DokkaConfigurationImpl(
-            moduleName = moduleName,
-            moduleVersion = moduleVersion.takeIf { it != "unspecified" },
-            outputDir = outputDirectory.asFile.get(),
-            pluginsClasspath = plugins.files.toList(),
-            sourceSets = listOf(buildDokkaSourceSetImpl(sources, classpath, taskPath))
-        )
-    }
+        override fun execute() {
+            val dokkaConfiguration =
+                buildDokkaConfiguration(
+                    parameters.dokkaPlugins,
+                    parameters.sources,
+                    parameters.classpath,
+                    parameters.outputDirectory,
+                    parameters.path.get(),
+                    parameters.projectPath.get(),
+                    parameters.moduleVersion.get())
 
-    private fun buildDokkaSourceSetImpl(
-        sources: ConfigurableFileCollection,
-        classpath: ConfigurableFileCollection,
-        taskPath: String
-    ): DokkaSourceSetImpl {
-
-        return DokkaSourceSetImpl(
-            sourceSetID = DokkaSourceSetID(scopeId = taskPath, sourceSetName = taskPath),
-            sourceRoots = sources.files ,
-            classpath = classpath.files.toList(),
-            externalDocumentationLinks = defaultExternalDocumentationLinks()
-        )
-    }
+            getRuntimeClassLoader(parameters.dokkaRuntimeClasspath).use {
+                val bootstrapClass = it.loadClass(DokkaBootstrapImpl::class.qualifiedName)
+                val bootstrapInstance = bootstrapClass.constructors.first().newInstance()
+                val configureMethod =
+                    bootstrapClass.getMethod("configure", String::class.java, BiConsumer::class.java)
+                val generateMethod = bootstrapClass.getMethod("generate")
 
-    private fun createProxyLogger(): BiConsumer<String, String> = BiConsumer { level, message ->
-        when (level) {
-            "debug" -> logger.debug(message)
-            "info" -> logger.info(message)
-            "progress" -> logger.lifecycle(message)
-            "warn" -> logger.warn(message)
-            "error" -> logger.error(message)
+                configureMethod.invoke(
+                    bootstrapInstance, dokkaConfiguration.toJsonString(), createProxyLogger())
+                generateMethod.invoke(bootstrapInstance)
+            }
+        }
+
+        /**
+         * Dokka needs a separate classpath because it not only uses classes bundled from the kotlin
+         * compiler, but also classes from intellij that would heavily interfere with other plugins
+         * when loaded directly.
+         *
+         * Given Gradle leaks kotlin-stdlib to classpath of workers running in classloader isolation
+         * and process isolation mode, we shouldn't launch Dokka via Gradle workers. Instead,
+         * we need to load Dokka runtime dependencies manually and launch it using reflection.
+         */
+        private fun getRuntimeClassLoader(runtime: ConfigurableFileCollection): URLClassLoader {
+            val runtimeJars = runtime.files
+            return URLClassLoader(
+                runtimeJars.map { it.toURI().toURL() }.toTypedArray(),
+                ClassLoader.getSystemClassLoader().parent
+            )
         }
-    }
 
-    private fun defaultExternalDocumentationLinks(): Set<ExternalDocumentationLinkImpl> {
-        val links = mutableSetOf<ExternalDocumentationLinkImpl>()
-        links.add(ExternalDocumentationLink.Companion.jdk(DokkaDefaults.jdkVersion))
-        links.add(ExternalDocumentationLink.Companion.kotlinStdlib())
-        links.add(ExternalDocumentationLink.Companion.androidSdk())
-        links.add(ExternalDocumentationLink.Companion.androidX())
-        return links
+        private fun buildDokkaConfiguration(
+            plugins: ConfigurableFileCollection,
+            sources: ConfigurableFileCollection,
+            classpath: ConfigurableFileCollection,
+            outputDirectory: DirectoryProperty,
+            taskPath: String,
+            moduleName: String,
+            moduleVersion: String
+        ): DokkaConfigurationImpl {
+            return DokkaConfigurationImpl(
+                moduleName = moduleName,
+                moduleVersion = moduleVersion.takeIf { it != "unspecified" },
+                outputDir = outputDirectory.asFile.get(),
+                pluginsClasspath = plugins.files.toList(),
+                sourceSets = listOf(buildDokkaSourceSetImpl(sources, classpath, taskPath))
+            )
+        }
+
+        private fun buildDokkaSourceSetImpl(
+            sources: ConfigurableFileCollection,
+            classpath: ConfigurableFileCollection,
+            taskPath: String
+        ): DokkaSourceSetImpl {
+
+            return DokkaSourceSetImpl(
+                sourceSetID = DokkaSourceSetID(scopeId = taskPath, sourceSetName = taskPath),
+                sourceRoots = sources.files ,
+                classpath = classpath.files.toList(),
+                externalDocumentationLinks = defaultExternalDocumentationLinks()
+            )
+        }
+
+        private fun createProxyLogger(): BiConsumer<String, String> = BiConsumer { level, message ->
+            val logger = Logging.getLogger("DokkaWorker")
+            when (level) {
+                "debug" -> logger.debug(message)
+                "info" -> logger.info(message)
+                "progress" -> logger.lifecycle(message)
+                "warn" -> logger.warn(message)
+                "error" -> logger.error(message)
+            }
+        }
+
+        private fun defaultExternalDocumentationLinks(): Set<ExternalDocumentationLinkImpl> {
+            val links = mutableSetOf<ExternalDocumentationLinkImpl>()
+            links.add(ExternalDocumentationLink.Companion.jdk(DokkaDefaults.jdkVersion))
+            links.add(ExternalDocumentationLink.Companion.kotlinStdlib())
+            links.add(ExternalDocumentationLink.Companion.androidSdk())
+            links.add(ExternalDocumentationLink.Companion.androidX())
+            return links
+        }
     }
 
     class CreationAction(
@@ -193,6 +230,10 @@ abstract class JavaDocGenerationTask : NonIncrementalTask() {
         override fun configure(task: JavaDocGenerationTask) {
             super.configure(task)
 
+            val dokkaParallelBuildService = getBuildService<DokkaParallelBuildService>(
+                creationConfig.services.buildServiceRegistry)
+            task.usesService(dokkaParallelBuildService)
+
             task.moduleVersion.setDisallowChanges(task.project.version.toString())
 
             val dokkaPluginConfig = task.project.configurations.detachedConfiguration(
@@ -207,6 +248,11 @@ abstract class JavaDocGenerationTask : NonIncrementalTask() {
             )
             task.dokkaRuntimeClasspath.fromDisallowChanges(runtimeConfig)
 
+            val dokkaCore = task.project.configurations.detachedConfiguration(
+                task.project.dependencies.create(DOKKA_CORE)
+            )
+            task.dokkaCoreClasspath.fromDisallowChanges(dokkaCore)
+
             task.sources.fromDisallowChanges(
                 creationConfig.sources.java.all,
                 creationConfig.sources.kotlin.all,
diff --git a/gmaven/src/test/resources/com/android/tools/test/gmaven-poms.txt b/gmaven/src/test/resources/com/android/tools/test/gmaven-poms.txt
index a891c25895..a9fde9fa76 100644
--- a/gmaven/src/test/resources/com/android/tools/test/gmaven-poms.txt
+++ b/gmaven/src/test/resources/com/android/tools/test/gmaven-poms.txt
@@ -483,7 +483,6 @@ com.android.tools.build:gradle
     io.grpc:grpc-stub (runtime)
     com.google.crypto.tink:tink (runtime)
     com.google.testing.platform:core-proto (runtime)
-    org.jetbrains.dokka:dokka-core (runtime)
     com.google.flatbuffers:flatbuffers-java (runtime)
     org.tensorflow:tensorflow-lite-metadata (runtime)