Machine-portable C/C++ path hashes

This change is a result of a meeting with androidx and reclient
(go/reclient) teams. The fact that hash paths change on a per-machine
basis is problematic because it would limit cache sharing.

This change makes the hashcode only depend on relative paths within
the project (along with other CMake/ndk-build flags).

I added a test that shows the path is stable between my local machine
and Linux, Mac, and Windows build-bots. My understanding is that
reclient uses Docker container linux build even for Windows and Mac.

Bug: NA
Test: Added a test

Change-Id: I132550e517a4d0241002af45b70c49a45c4d70da

5 files changed, 129 insertions, 13 deletions

diff --git a/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/CreateCxxAbiModel.kt b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/CreateCxxAbiModel.kt
index 2f90297992..4a2d77bddc 100644
--- a/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/CreateCxxAbiModel.kt
+++ b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/CreateCxxAbiModel.kt
@@ -88,6 +88,7 @@ fun createCxxAbiModel(
                 },
                 buildSettings = createBuildSettingsFromFile(module.buildSettingsFile),
                 fullConfigurationHash = configurationHash,
+                fullConfigurationHashKey = "",
                 configurationArguments = listOf(),
                 isActiveAbi = validAbiList.contains(abi),
                 prefabFolder = join(variantCxxBuildFolder, "prefab", abi.tag),
diff --git a/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/CxxAbiModel.kt b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/CxxAbiModel.kt
index fda7ba770b..73b4f2415a 100644
--- a/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/CxxAbiModel.kt
+++ b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/CxxAbiModel.kt
@@ -99,6 +99,11 @@ data class CxxAbiModel(
     val fullConfigurationHash: String,
 
     /**
+     * The string value used to calculate [fullConfigurationHash]
+     */
+    val fullConfigurationHashKey: String,
+
+    /**
      * The inputs used to compute [fullConfigurationHash]
      */
     val configurationArguments: List<String>,
@@ -440,6 +445,13 @@ val CxxAbiModel.platformCode
     } ?: ""
 
 /**
+ * File that holds the key for the hashed subfolder.
+ *   ex, $moduleRootFolder/.cxx/Debug/{hashcode}/hash_key.txt
+ */
+val CxxAbiModel.cxxBuildHashKeyFile : File get() =
+    cxxBuildFolder.parentFile.resolve("hash_key.txt")
+
+/**
  * Construct a ninja command-line with [args] at the end.
  */
 fun CxxAbiModel.createNinjaCommand(args: List<String>) : List<String> {
diff --git a/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/JsonUtil.kt b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/JsonUtil.kt
index 77787f90d5..2513402358 100644
--- a/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/JsonUtil.kt
+++ b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/model/JsonUtil.kt
@@ -16,7 +16,9 @@
 
 package com.android.build.gradle.internal.cxx.model
 
+import com.android.build.gradle.internal.cxx.io.writeTextIfDifferent
 import com.android.build.gradle.internal.cxx.json.PlainFileGsonTypeAdaptor
+import com.android.build.gradle.internal.cxx.json.jsonStringOf
 import com.android.repository.Revision
 import com.google.gson.GsonBuilder
 import com.google.gson.TypeAdapter
@@ -83,6 +85,8 @@ fun createCxxAbiModelFromJson(json: String): CxxAbiModel {
 fun CxxAbiModel.writeJsonToFile() {
     modelOutputFile.parentFile.mkdirs()
     modelOutputFile.writeText(toJsonString())
+    cxxBuildHashKeyFile.parentFile.mkdirs()
+    cxxBuildHashKeyFile.writeTextIfDifferent(fullConfigurationHashKey)
 }
 
 /**
diff --git a/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/settings/CxxAbiModelSettingsRewriter.kt b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/settings/CxxAbiModelSettingsRewriter.kt
index 6c8b0b60fd..3c2791941d 100644
--- a/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/settings/CxxAbiModelSettingsRewriter.kt
+++ b/build-system/gradle-core/src/main/java/com/android/build/gradle/internal/cxx/settings/CxxAbiModelSettingsRewriter.kt
@@ -90,6 +90,7 @@ import com.google.common.collect.Lists
 import org.gradle.api.file.ProjectLayout
 import org.gradle.api.provider.ProviderFactory
 import java.io.File
+import com.android.Version
 
 /**
  * Rewrite the CxxAbiModel in three phases:
@@ -189,27 +190,78 @@ private fun CxxAbiModel.supportNdkR14AndEarlier() : CxxAbiModel {
 }
 
 /**
- * Calculate hash and plug it into the model.
+ * Calculate hash and plug it into the model. Return the new model along with the text that
+ * was hashed.
+ *
+ * The user can pass arbitrary parameters to CMake (or ndk-build) so we can't guard against every
+ * case that could make the hash local-machine-specific. This is best effort, works in the default
+ * case, and we provide the hash_key.txt file (written later) to be able to diagnose problems.
+ * The consequence of a non-portable paths is that hypothetical caching tech (ccache, reclient,
+ * gomacc, etc) may not be able to share results. However, build results will still be accurate.
  */
-private fun CxxAbiModel.calculateConfigurationHash() =
-    copy(fullConfigurationHash = sha256Of(configurationArguments))
+private fun CxxAbiModel.calculateConfigurationHash() : CxxAbiModel {
+    var header =
+        """
+        # Values used to calculate the hash in this folder name.
+        # Should not depend on the absolute path of the project itself.
+        #   - AGP: ${Version.ANDROID_GRADLE_PLUGIN_VERSION}.
+        #   - ${'$'}NDK is the path to NDK ${variant.module.ndkVersion}.
+        #   - ${'$'}PROJECT is the path to the parent folder of the root Gradle build file.
+        #   - ${'$'}ABI is the ABI to be built with. The specific value doesn't contribute to the value of the hash.
+        #   - ${'$'}HASH is the hash value computed from this text.
+
+        """.trimIndent()
+
+    var arguments = configurationArguments.joinToString("\n")
+        .replace(variant.module.ndkFolder.path, "\$NDK")
+        .replace(variant.module.project.rootBuildGradleFolder.path, "\$PROJECT")
+        .replace(NDK_ABI.ref, "\$ABI")
+        .replace(NDK_CONFIGURATION_HASH.ref, "\$HASH")
+
+    if (variant.module.cmake != null) {
+        header +=             """
+            #   - ${'$'}CMAKE is the path to CMake ${variant.module.cmake.minimumCmakeVersion}.
+
+            """.trimIndent()
+        arguments = arguments.replace(variant.module.cmake.cmakeExe!!.path, "\$CMAKE")
+
+    }
+
+    if (variant.module.ninjaExe != null) {
+        header +=             """
+            #   - ${'$'}NINJA is the path to Ninja.
+
+            """.trimIndent()
+        arguments = arguments.replace(variant.module.ninjaExe.path, "\$NINJA")
+    }
+
+    val hashKey = header + arguments.replace("\\", "/")
+
+    return copy(
+        fullConfigurationHash = sha256Of(hashKey, includeGradleVersionInHash = false),
+        fullConfigurationHashKey = hashKey
+    )
+}
 
 /**
  * Finally, expand all of the post-hash macros.
  */
-private fun CxxAbiModel.expandConfigurationHashMacros() = rewrite { _, value ->
-    reifyString(value) { tokenMacro ->
-        when(tokenMacro) {
-            NDK_ABI.qualifiedName,
-            NDK_CONFIGURATION_HASH.qualifiedName,
-            NDK_FULL_CONFIGURATION_HASH.qualifiedName -> {
-                val macro = Macro.lookup(tokenMacro) ?: error("Unrecognized macro: $tokenMacro")
-                resolveMacroValue(macro)
+private fun CxxAbiModel.expandConfigurationHashMacros() : CxxAbiModel {
+    val result = rewrite { _, value ->
+        reifyString(value) { tokenMacro ->
+            when(tokenMacro) {
+                NDK_ABI.qualifiedName,
+                NDK_CONFIGURATION_HASH.qualifiedName,
+                NDK_FULL_CONFIGURATION_HASH.qualifiedName -> {
+                    val macro = Macro.lookup(tokenMacro) ?: error("Unrecognized macro: $tokenMacro")
+                    resolveMacroValue(macro)
+                }
+                else ->
+                    error(tokenMacro)
             }
-            else ->
-                error(tokenMacro)
         }
     }
+    return result
 }
 
 /**
diff --git a/build-system/integration-test/native/src/test/java/com/android/build/gradle/integration/nativebuild/CmakeBasicProjectTest.kt b/build-system/integration-test/native/src/test/java/com/android/build/gradle/integration/nativebuild/CmakeBasicProjectTest.kt
index 8f5e0ad177..0d31e4aa8e 100644
--- a/build-system/integration-test/native/src/test/java/com/android/build/gradle/integration/nativebuild/CmakeBasicProjectTest.kt
+++ b/build-system/integration-test/native/src/test/java/com/android/build/gradle/integration/nativebuild/CmakeBasicProjectTest.kt
@@ -17,7 +17,9 @@
 package com.android.build.gradle.integration.nativebuild
 
 import com.android.SdkConstants.CURRENT_PLATFORM
+import com.android.SdkConstants.NDK_DEFAULT_VERSION
 import com.android.SdkConstants.PLATFORM_WINDOWS
+import com.android.Version
 import com.android.build.gradle.integration.common.fixture.GradleBuildResult
 import com.android.build.gradle.integration.common.fixture.GradleTestProject
 import com.android.build.gradle.integration.common.fixture.GradleTestProject.ApkLocation
@@ -53,11 +55,13 @@ import com.android.build.gradle.internal.core.Abi
 import com.android.build.gradle.internal.cxx.attribution.decodeBuildTaskAttributions
 import com.android.build.gradle.internal.cxx.configure.CMakeVersion
 import com.android.build.gradle.internal.cxx.configure.shouldConfigure
+import com.android.build.gradle.internal.cxx.hashing.sha256Of
 import com.android.build.gradle.internal.cxx.io.SynchronizeFile.Outcome.CREATED_HARD_LINK_FROM_SOURCE_TO_DESTINATION
 import com.android.build.gradle.internal.cxx.io.decodeSynchronizeFile
 import com.android.build.gradle.internal.cxx.json.AndroidBuildGradleJsons
 import com.android.build.gradle.internal.cxx.json.AndroidBuildGradleJsons.getNativeBuildMiniConfig
 import com.android.build.gradle.internal.cxx.model.compileCommandsJsonBinFile
+import com.android.build.gradle.internal.cxx.model.cxxBuildHashKeyFile
 import com.android.build.gradle.internal.cxx.model.jsonGenerationLoggingRecordFile
 import com.android.build.gradle.internal.cxx.model.miniConfigFile
 import com.android.build.gradle.internal.cxx.model.ninjaBuildFile
@@ -411,6 +415,49 @@ class CmakeBasicProjectTest(
     }
 
     @Test
+    fun `ensure hashed output paths are stable`() {
+        Assume.assumeTrue(mode == Mode.CMake && cmakeVersionInDsl != "3.6.0")
+        project.execute("configure${mode.buildFolderTag}Debug[x86_64]")
+        val abi = project.recoverExistingCxxAbiModels().single { it.abi == Abi.X86_64 }
+        val hashKey = abi.cxxBuildHashKeyFile.readText()
+        val hashSegment = abi.cxxBuildHashKeyFile.parentFile.name
+        val hashKeyExpected =
+            """
+            # Values used to calculate the hash in this folder name.
+            # Should not depend on the absolute path of the project itself.
+            #   - AGP: ${Version.ANDROID_GRADLE_PLUGIN_VERSION}.
+            #   - ${'$'}NDK is the path to NDK $NDK_DEFAULT_VERSION.
+            #   - ${'$'}PROJECT is the path to the parent folder of the root Gradle build file.
+            #   - ${'$'}ABI is the ABI to be built with. The specific value doesn't contribute to the value of the hash.
+            #   - ${'$'}HASH is the hash value computed from this text.
+            #   - ${'$'}CMAKE is the path to CMake $cmakeVersionInDsl.
+            #   - ${'$'}NINJA is the path to Ninja.
+            -H${'$'}PROJECT
+            -DCMAKE_SYSTEM_NAME=Android
+            -DCMAKE_EXPORT_COMPILE_COMMANDS=ON
+            -DCMAKE_SYSTEM_VERSION=16
+            -DANDROID_PLATFORM=android-16
+            -DANDROID_ABI=${'$'}ABI
+            -DCMAKE_ANDROID_ARCH_ABI=${'$'}ABI
+            -DANDROID_NDK=${'$'}NDK
+            -DCMAKE_ANDROID_NDK=${'$'}NDK
+            -DCMAKE_TOOLCHAIN_FILE=${'$'}NDK/build/cmake/android.toolchain.cmake
+            -DCMAKE_MAKE_PROGRAM=${'$'}NINJA
+            -DCMAKE_C_FLAGS=-DTEST_C_FLAG -DTEST_C_FLAG_2
+            -DCMAKE_CXX_FLAGS=-DTEST_CPP_FLAG
+            -DCMAKE_LIBRARY_OUTPUT_DIRECTORY=${'$'}PROJECT/build/intermediates/cxx/Debug/${'$'}HASH/obj/${'$'}ABI
+            -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=${'$'}PROJECT/build/intermediates/cxx/Debug/${'$'}HASH/obj/${'$'}ABI
+            -DCMAKE_BUILD_TYPE=Debug
+            -B${'$'}PROJECT/.cxx/Debug/${'$'}HASH/${'$'}ABI
+            -GNinja
+            """.trimIndent()
+        assertThat(hashKey).isEqualTo(hashKeyExpected)
+        val expectedHashSegment = sha256Of(hashKeyExpected, includeGradleVersionInHash = false).substring(0,8)
+        // If the text above passes then the SHA-256 of it should be stable.
+        assertThat(hashSegment).isEqualTo(expectedHashSegment)
+    }
+
+    @Test
     fun `ensure CMake arguments have macros expanded`() {
         if (mode != Mode.CMake) return // This is a CMake-only test
         TestFileUtils.appendToFile(